Changeset 28017 for branches/czw_branch/20100427/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

Timestamp:

May 18, 2010, 5:06:01 PM (16 years ago)

Author:

watersc1

Message:

Almost working version of the mask-stats/software revision code.

Almost.

Location:

branches/czw_branch/20100427

Files:

• . (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (34 diffs)

branches/czw_branch/20100427/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -17 +17 @@
                     resolve_project
                     getCamRunByCamID
+                    parse_md_fast
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -27 +28 @@
 use PS::IPP::Config qw( :standard );
 use Carp;
-use DateTime;
 use File::Temp qw(tempfile);
 use File::Basename;
 use POSIX;
+use Time::HiRes qw(gettimeofday);
 
 my $dvo_verbose = 0;
@@ -169 +170 @@
 
 # The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are
-# triggered by a bydiff request
-
-sub lookup
-{
+# triggered by a bydiff or bycoord request
+
+sub lookup {
     my $ipprc    = shift;
     my $rowList    = shift;
@@ -206 +206 @@
     my $class_id;
     my $skycell_id;
-
-    if (isnull($component)) {
-        $component = undef;
-    }
+    my $default_error = $PSTAMP_NO_IMAGE_MATCH;
 
     # note $magic_arg may be cleared below depending on $req_type
@@ -220 +217 @@
     my $choose_components = 0;
     if ($stage eq "raw") {
-        $class_id = $component;
         $command = "$regtool -processedimfile -dbname $imagedb";
         # XXX: for now restrict lookups to type object 
@@ -226 +222 @@
         $command .= " -exp_type object";
         $id_opt = $use_imfile_id ? "-raw_imfile_id" : "-exp_id";
-        $component_args = " -class_id $class_id" if $component;
+
+        if ($component and $component ne 'all') {
+            $class_id = $component;
+            $component_args = " -class_id $class_id";
+        }
         $want_astrom = 1;
         $set_class_id = 1;
     } elsif ($stage eq "chip") {
-        $class_id = $component;
         # if the request is such that it will yield a single image per "run" or the
         # center is specified in pixel coordinates 
         # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
         # choose the chips containing the center pixel by calling selectComponets() below
-        if ($class_id or $use_imfile_id or !$skycenter) {
+        if ($component or $use_imfile_id or !$skycenter) {
             $command = "$chiptool -processedimfile -dbname $imagedb";
-            $component_args = " -class_id $class_id" if $class_id;
+            if ($component and $component ne 'all') {
+                $class_id = $component;
+                $component_args = " -class_id $class_id";
+            }
         } else {
             $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
@@ -246 +248 @@
         # With the mdc file it has all of the information that it needs.
         $image_name   = "PPIMAGE.CHIP";
-        $mask_name    = "PPIMAGE.CHIP.MASK";
+        # $mask_name    = "PPIMAGE.CHIP.MASK";
+        # XXX: really need to handle this properly!
+        $mask_name    = "PSASTRO.OUTPUT.MASK";
         $weight_name  = "PPIMAGE.CHIP.VARIANCE";
         $cmf_name     = "PSPHOT.OUTPUT";
@@ -255 +259 @@
         $set_class_id = 1;
     } elsif ($stage eq "warp") {
-        $skycell_id = $component;
-        if ($skycell_id or $use_imfile_id or !$skycenter) {
+        if ($component or $use_imfile_id or !$skycenter) {
             $command = "$warptool -warped -dbname $imagedb";
-            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+            if ($component and $component ne 'all') {
+                $skycell_id = $component;
+                $component_args = " -skycell_id $skycell_id";
+            }
         } else {
             $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
@@ -271 +277 @@
         $base_name    = "path_base"; # name of the field for the warptool output
     } elsif ($stage eq "diff") {
-        $skycell_id = $component;
-        if ($skycell_id or $use_imfile_id or !$skycenter) {
+        if ($component or $use_imfile_id or !$skycenter) {
             $command = "$difftool -diffskyfile -dbname $imagedb";
-            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+            if ($component and $component ne 'all') {
+                $skycell_id = $component;
+                $component_args = " -skycell_id $skycell_id";
+            }
         } else {
             $command = "$difftool -listrun -pstamp_order -dbname $imagedb";
@@ -284 +292 @@
         $weight_name = "PPSUB.OUTPUT.VARIANCE";
         $cmf_name    = "PPSUB.OUTPUT.SOURCES";
-        $psf_name     = "PSPHOT.PSF.SKY.SAVE";
+        $psf_name    = "PSPHOT.PSF.SKY.SAVE";
         $base_name   = "path_base";
     } elsif ($stage eq "stack") {
@@ -361 +369 @@
         $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
     } else {
-        # put each of the rows into all of the the images
+        # put index for each of the rows into all of the the images
         setRowRefs($rowList, $images);
     }
@@ -390 +398 @@
             $ipprc->define_camera($camera);
         }
+        # ok it's a keeper
         my $out = $image;
 
         # if uri is nil this will get overridded below
         # (we do this here for raw stage)
-        $out->{image}  = $image->{uri};
+        $out->{image}  = $out->{uri};
 
         if ($set_class_id) {
-            $class_id = $image->{class_id};
+            $class_id = $out->{class_id};
             $out->{component} = $class_id;
         } else {
-            $out->{component} = $image->{skycell_id};
+            $out->{component} = $out->{skycell_id};
         }
         my $stage_id;
         if ($stage eq "raw") {
-            $stage_id = $image->{exp_id};
+            $stage_id = $out->{exp_id};
         } elsif ($stage eq "chip") {
-            $stage_id = $image->{chip_id};
+            $stage_id = $out->{chip_id};
         } elsif ($stage eq "warp") {
-            $stage_id = $image->{warp_id};
+            $stage_id = $out->{warp_id};
         } elsif ($stage eq "diff") {
-            $stage_id = $image->{diff_id};
-            if ($inverse && $image->{bothways}) {
+            $stage_id = $out->{diff_id};
+            if ($inverse && $out->{bothways}) {
                 $image_name  = "PPSUB.INVERSE";
                 $mask_name   = "PPSUB.INVERSE.MASK";
@@ -418 +427 @@
             }
         } elsif ($stage eq "stack") {
-            $stage_id = $image->{stack_id};
+            $stage_id = $out->{stack_id};
         }
         $out->{stage_id} = $stage_id;
@@ -424 +433 @@
         $out->{imagedb} = $imagedb;
 
+        my $mask_base;
+        if ($want_astrom) {
+            if (! defined $out->{astrom}) {
+                if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) {
+                    print STDERR "failed to find astrometry for $stage $stage_id\n";
+                    next;
+                }
+            }
+            # XXX: do this right by looking at the recipe
+            $mask_base = $out->{cam_path_base};
+        } else {
+            $mask_base = $base;
+        }
+
         if ($base) {
             $out->{image}  = $ipprc->filename($image_name,  $base, $class_id) if $image_name;
-            $out->{mask}   = $ipprc->filename($mask_name,   $base, $class_id) if $mask_name;
+            $out->{mask}   = $ipprc->filename($mask_name,   $mask_base, $class_id) if $mask_name;
             $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
-            $out->{cmf}    = $ipprc->filename($cmf_name, $base, $class_id) if $cmf_name;
-            $out->{psf}    = $ipprc->filename($psf_name, $base, $class_id) if $psf_name;
-            $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name;
-        }
-        if ($want_astrom and ! defined $out->{astrom}) {
-            $out->{astrom} = find_astrometry($ipprc, $imagedb, $image, $verbose);
+            $out->{cmf}    = $ipprc->filename($cmf_name,    $base, $class_id) if $cmf_name;
+            $out->{psf}    = $ipprc->filename($psf_name,    $base, $class_id) if $psf_name;
+            $out->{backmdl}= $ipprc->filename($backmdl_name,$base, $class_id) if $backmdl_name;
         }
 
@@ -455 +475 @@
     my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
 
-    my $command = "$difftool -diffskyfile -dbname $imagedb";
+    my $command = "$difftool -dbname $imagedb";
     
+    my $listrun = 0;
     if ($byid) {
-        $command .= " -diff_id $id";
+        if ($skycell_id) {
+            $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id";
+        } else {
+            $command .= " -listrun -diff_id $id";
+            $listrun = 1;
+        }
     } else {
-        $command .= " -diff_skyfile_id $id";
-    }
-    $command .= " -skycell_id $skycell_id" if $skycell_id;
-
-    my $images = runToolAndParse($command, $verbose);
-
-    my $n = @$images;
-    if (!$byid && ($n > 1)) {
+        $command .= " -diffskyfile -diff_skyfile_id $id";
+    }
+
+    my $output = runToolAndParse($command, $verbose);
+
+    my $n = $output ? scalar @$output : 0;
+    if (!$listrun && ($n > 1)) {
         die ("difftool returned an unexpected number of diffskyfiles: $n");
     } elsif ($n == 0) {
         return undef;
+    }
+
+    my $images;
+    if ($listrun) {
+        $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose);
+    } else {
+        $images = $output;
     }
 
@@ -501 +533 @@
             if (($img_type ne "diff") and ($img_type ne "stack")) {
                 print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n";
-                next;
+                setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST);
+                # all images will be the same so we can stop
+                last;
             }
             # stack-stack diff
@@ -569 +603 @@
             } else {
                 # XXX this will only happen if the minuend is not a warp. See hack above
-                print STDERR "WARNING: cannot resolve camera so cannot get resolve file rules\n";
+                print STDERR "WARNING: cannot resolve camera so cannot find file rules\n";
                 next;
             }
@@ -691 +725 @@
         next if !$astrom_resolved;
 
-        my $start_dvo = DateTime->now->mjd;
+        my $start_dvo = gettimeofday();
         my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec";
         # run the tool and parse the output
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $dvo_verbose);
-        my $dtime_dvo = (DateTime->now->mjd - $start_dvo)* 86400.;
+        my $dtime_dvo = gettimeofday() - $start_dvo;
 #        print "Time to run dvoImagesAtCoords: $dtime_dvo\n";
         unless ($success) {
@@ -720 +754 @@
         my $n = scalar @lines;
         if ($n != 1) {
-            print STDERR "unexpected number of lines returned by dvoImagesAtCoords: $n\n";
-
             # XXX: There is a bug in dvo where each component is listed twice
             # When that gets fixed remove the conditional and just die
-            die "unexpected number lines returned by dvoImagesAtCoords: $n"
-                if ($n != 2) or ($lines[0] ne $lines[1]);
+            if ($n ne 2 or $lines[0] ne $lines[1]) {
+                print STDERR "unexpected number of lines returned by dvoImagesAtCoords: $n\n";
+                print STDERR "OUTPUT:\n$output\n";
+                # actually this seems to happen sometimes. Probably due to a problem with
+                # astrometry. Just skip this camRun
+                print STDERR "skipping: $astrom\n";
+                next;
+            }
         }
 
@@ -741 +779 @@
             chip_id   => $camRun->{chip_id},
             cam_id    => $camRun->{cam_id},
-            astrom    => $astrom_resolved,
+            astrom    => $astrom_resolved,  # XXX: is astrom used?
             class_id  => $class_id,
             component => $class_id
@@ -821 +859 @@
 # cache of results of ppConfigDump
 my %astromSources;
-my $last_exp_id = 0;
-my $lastAstromFile;
 
 # find the astrometry file for a given exposure
 # return undef if no completed camRun exists
+# XXX Right now this probably only works cameras where the astrometry is done at the camera stage
+# What other possibilities are there for ASTROM.SOURCE besides PSASTRO.OUTPUT?
+
 sub find_astrometry {
     my $ipprc = shift;
@@ -835 +874 @@
 
     my $exp_id = $image->{exp_id};
-    if (($exp_id eq $last_exp_id) and $lastAstromFile) {
-        # running camtool 60 times is really expensive when the answer is the same every time
-        return $lastAstromFile;
-    }
-    $last_exp_id = 0;
-    $lastAstromFile = undef;
 
     my $camRun = getCamRunByExpID($exp_id);
@@ -850 +883 @@
         unless ($success) {
             print STDERR @$stderr_buf;
-            return undef;
+            return 0;
         }
 
@@ -858 +891 @@
         if (!$output) {
             print STDERR "no output returned from $command\n" if $verbose;
-            return undef;
+            return 0;
         }
         my $camruns = parse_md_fast($mdcParser, $output);
         if (!$camruns) {
-            return undef;
+            return 0;
         }
 
         # If there are multiple cam runs for this exposure, take the last completed one with good quality
         # on the assumption that it has the best astrometry.
-        foreach $camRun (@$camruns) {
-            last if (($camRun->{state} eq 'full') and ($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
+        foreach my $cr (@$camruns) {
+            if (($cr->{state} eq 'full') and ($cr->{quality} eq 0) and ($cr->{fault} eq 0)) {
+                $camRun = $cr;
+                last;
+            }
         }
         # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT
         if (!$camRun) {
             # no cam runs for this exposure id therefore best astrometry is whatever is in the header
-            return undef;
+            return 0;
         }
         updateCamRunCache($camRun);
@@ -902 +938 @@
     # XXX: Is this code correct if ASTROM.SOURCE ne "PSASTRO.OUTPUT"
     my $astromFile = $ipprc->filename($astromSource, $camRoot);
-    if ($astromFile) {
-        $lastAstromFile = $astromFile;
-        $last_exp_id = $exp_id;
-    }
-
-    return $astromFile;
+    if (!$astromFile) {
+        print STDERR "failed to find astrometry file from $astromSource $camRoot\n";
+        return 0;
+    }
+
+    $image->{astrom} = $astromFile;
+    $image->{cam_path_base} = $camRoot;
+
+    return 1;
 }
 
@@ -973 +1012 @@
     }
 
-    if ($project_name eq $last_project) {
+    if ($last_project and ($project_name eq ($last_project->{name}))) {
         return $last_project;
     }
@@ -1049 +1088 @@
         print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n";
         $npoints++;
+        # this gets overwitten if an overlapping image is found
+        $row->{error_code} = $PSTAMP_NO_OVERLAP;
     }
     close $pointsList;
@@ -1093 +1134 @@
                 }
                 push @$ref, $ptnum;
+                my $row = $rowList->[$ptnum];
+                # this row found a match
+                $row->{error_code} = 0;
             }
             if ($verbose) {
@@ -1124 +1168 @@
                 my $images = runToolAndParse($command, $verbose);
                 if (!defined $images) {
-                    # XXX: need to get the set error_code to $PSTAMP_NO_OVERLAP
                     print "No components containing coordinates found for ${stage}_id $stage_id\n";
                     foreach my $row (@$rowList) {
+                        # XXX: This doesn't seem like the correct error code?
                         $row->{error_code} = $PSTAMP_NO_OVERLAP;
                     }
@@ -1221 +1265 @@
 
     print "Running $command\n" if !$verbose;
-    my $start_tool = DateTime->now->mjd;
+    my $start_tool = gettimeofday();
     # run the command and parse the output
 
@@ -1232 +1276 @@
     }
 
-    my $now = DateTime->now->mjd;
-    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
+    my $now = gettimeofday();
+    my $dtime_tool = $now - $start_tool;
     print "Time to run $program: $dtime_tool\n";
 
@@ -1241 +1285 @@
     }
 
-    my $start_parse = DateTime->now->mjd;
+    my $start_parse = gettimeofday();
 
     my $mdcParser = PS::IPP::Metadata::Config->new;
@@ -1247 +1291 @@
         or die ("Unable to parse metadata config doc");
 
-    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
+    my $dtime_parse = gettimeofday() - $start_parse;
     print "Time to parse results from $program: $dtime_parse\n";
 
@@ -1287 +1331 @@
     }
 }
+# set error_code for an array of rows to a given value
+sub setErrorCodes {
+    my $rowList = shift;
+    my $code = shift;
+    foreach my $row (@$rowList) {
+        $row->{error_code} = $code;
+    }
+}
 
 sub my_die