Changeset 36680 for branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp

Timestamp:

Apr 21, 2014, 5:42:34 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (20 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (7 diffs)

branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
 use POSIX;
 use Time::HiRes qw(gettimeofday);
+use IO::Handle;
 
 my $dvo_verbose = 0;
@@ -176 +177 @@
         $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
         0, 0,   # fwhm cuts are not applied here
-        undef, undef, undef, $verbose);
+        undef, 0, # no cam run information
+        undef, undef, $verbose);
 
     return $results;
@@ -243 +245 @@
     if (isnull($filter)) {
         $filter = undef;
+    }
+
+    if ($stage eq 'stack_summary') {
+        # stack_summary jobs are so different from others that we lookup in a specialized function
+        return lookup_stack_summary($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
     }
 
@@ -322 +329 @@
         0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
         getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}),
-        undef, $release_name, $survey,
+        undef, 0, $release_name, $survey,
         $verbose);
 
@@ -351 +358 @@
     my $fwhm_max = shift;
     my $selectedAstrom = shift;
+    my $selected_cam_id = shift;
     my $release_name = shift;
     my $survey = shift;
@@ -460 +468 @@
             $skycell_id = $component;
             $choose_components = 1;
+            if ($req_type eq 'byskycell' and $skycenter) {
+                # Add a wide coordinate cut to the releasetool command.
+                # This greatly speeds up the query by reducing the number of exposures considered
+                # to those near the region of interest.
+                # The actual radius value that we use is not critical because the skycell cut does the relevant
+                # selection.
+                $command .= " -ra $row->{CENTER_X} -decl $row->{CENTER_Y} -radius 5.0";
+            }
         } else {
             $command = "$warptool -pstamp_order -dbname $imagedb";
@@ -602 +618 @@
         # The image selectors are such that multiple runs my have be returned for the same exposure.
         # Return only the latest one.
-        $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
+        $images = filterRuns($stage, $choose_components, $need_magic, $images, $inverse, $verbose);
     }
 
@@ -685 +701 @@
                 $out->{data_state} = $out->{state} = $out->{stack_state};
             } else {
-                # XXX: Consider looking up skycal results even if we are
+                $out->{data_state} = $out->{state};
+
+                # XXX: Consider looking up skycal and staticsky results even if we are
                 # not using releasetool
-                $out->{data_state} = $out->{state};
             }
         }
@@ -699 +716 @@
                 $out->{astrom} = $selectedAstrom;
                 $out->{cam_path_base} = $selectedAstrom;
+                $out->{cam_id} = $selected_cam_id if $selected_cam_id;
                 if ($selectedAstrom =~ /\.smf$/) {
                     $out->{cam_path_base} =~ s/\.smf$//;
@@ -722 +740 @@
             $out->{mask}   = $ipprc->filename($mask_name,   $mask_base, $class_id) if $mask_name;
             $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
-            $out->{cmf}    = $ipprc->filename($cmf_name,    $base, $class_id) if $cmf_name;
-            $out->{psf}    = $ipprc->filename($psf_name,    $base, $class_id) if $psf_name;
             $out->{backmdl}= $ipprc->filename($backmdl_name,$base, $class_id) if $backmdl_name;
+            if ($image->{staticsky_path_base}) {
+                $out->{psf}    = $ipprc->filename("PSPHOT.STACK.PSF.SAVE", $image->{staticsky_path_base}, $image->{stack_id});
+                # if astrom was not set above to the skycal cmf get the sources file from staticsky
+                $out->{cmf}    = $ipprc->filename("PSPHOT.STACK.OUTPUT",   $image->{staticsky_path_base}, $image->{stack_id}) if !$out->{astrom};
+            } else {
+                $out->{psf}    = $ipprc->filename($psf_name,    $base, $class_id) if $psf_name;
+                $out->{cmf}    = $ipprc->filename($cmf_name,    $base, $class_id) if $cmf_name;
+            }
         }
 
@@ -731 +755 @@
 
     return $output;
+}
+
+sub lookup_stack_summary {
+    my ($ipprc, $row, $imagedb, $tess_id, $component, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose) = @_;
+
+    $row->{error_code} = $PSTAMP_NO_IMAGE_MATCH;
+
+    $component = '' if $component eq 'all';
+
+    my $req_type = $row->{REQ_TYPE};
+    my $results;
+    if ($req_type eq 'byid') {
+        my $sass_id = $row->{ID};
+        my $command = "$stacktool -dbname $imagedb -summary -sass_id $sass_id";
+        $results = runToolAndParse($command, $verbose);
+    } else {
+        my ($release_name, $survey, $default_tess_id) = get_release_info($row);
+        if (!$tess_id and $default_tess_id) {
+            $tess_id = $default_tess_id;
+        }
+
+        my $skycells;
+        if ($req_type eq 'bycoord') {
+            my $ra = $row->{CENTER_X};
+            my $dec = $row->{CENTER_Y};
+            $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose);
+        } elsif ($req_type eq 'byskycell') {
+            if (!defined $tess_id or !defined $component) {
+                print STDERR "Error: TESS_ID and  COMPONENT are required for REQ_TYPE byskycell\n";
+                $row->{error_code} = $PSTAMP_INVALID_REQUEST;
+            }
+            my $skycell = {
+                tess_id    => $tess_id,
+                component => $component
+            };
+            push @$skycells, $skycell
+        } else {
+            print STDERR "Error: $req_type is not a valid REQ_TYPE for IMG_TYPE stack_summary\n";
+            $row->{error_code} = $PSTAMP_INVALID_REQUEST;
+            return undef;
+        }
+
+        if ($skycells) {
+            # We have a list of skycells. Find matching projection cells.
+            # and then matching rows in stackSummary
+
+            my %projection_cells_found;
+            foreach my $skycell (@$skycells) {
+                my $tess_id = $skycell->{tess_id};
+                my $skycell_id = $skycell->{component};
+
+                my $projection_cell = skycell_id_to_projection_cell($skycell_id);
+
+                # only need to include a projection cell once
+                next if $projection_cells_found{$projection_cell};
+
+                $projection_cells_found{$projection_cell} = 1;
+
+                # print "$tess_id $skycell_id $projection_cell\n";
+
+                my $command;
+                if ($release_name or $survey) {
+                    $command = "$releasetool -dbname $imagedb";
+                    if ($release_name) {
+                        $command .= " -release_name $release_name";
+                    } else {
+                        $command .= " -priority_order";
+                    }
+                    $command .= " -surveyName $survey" if $survey;
+                } else {
+                    $command = "$stacktool -dbname $imagedb";
+                }
+                $command .= " -summary -tess_id $tess_id";
+                $command .= " -projection_cell $projection_cell";
+
+                $command .= " -filter $filter" if $filter;
+                $command .= " -data_group $data_group" if $data_group;
+
+                my $these_results = runToolAndParse($command, $verbose);
+                push @$results, @$these_results if $these_results;
+            }
+        }
+    }
+
+    if ($results) {
+        foreach my $summary (@$results) {
+            my $path_base = $summary->{path_base};
+            my $projection_cell = $summary->{projection_cell};
+
+            # we need to flesh out the hashes returned with values that the parser expects
+            $summary->{stage} = 'stack_summary';
+            $summary->{component} = $projection_cell;
+            $summary->{stage_id} = $summary->{sass_id};
+            $summary->{path_base} = "$path_base.$projection_cell";
+
+            # The stack_summary stage doesn't follow the usual ipp conventions for file rules
+            # and path_base. We leave it up to the program that runs the job to handle this.
+            # However, the parser uses the image parameter to set the job's outputBase (by removing the .fits)
+            # so we need to set it. Just use the new path_base.
+            $summary->{image} = $summary->{path_base};
+            $summary->{state} = 'full';
+            $summary->{data_state} = 'full';
+            $summary->{fault} = 0;
+            $summary->{imagedb} = $imagedb;
+            $summary->{row_index} = [];
+            push @{$summary->{row_index}}, 0;
+
+            # XXX: Are there any other required parameters? 
+            # TODO: create a function to validate the components returned
+            # and make sure that any required elements are there.
+        }
+    }
+
+    return $results;
 }
 
@@ -946 +1084 @@
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max,
                     $fwhm_min, $fwhm_max,
-                    $chip->{astrom}, undef, undef,
+                    $chip->{astrom}, $chip->{cam_id}, undef, undef,
                     $verbose);
 
@@ -964 +1102 @@
             setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH);
         } else {
-            # XXX: We are not applying the fwhm cuts
+            # XXX: We are not applying the fwhm cuts unless we use releasetool
             foreach my $skycell (@$skycells) {
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef,
@@ -970 +1108 @@
                     $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 
                     $fwhm_min, $fwhm_max,
-                    undef, $release_name, $survey,
+                    undef, 0, $release_name, $survey,
                     $verbose);
 
@@ -1166 +1304 @@
 
     my @lines;
-    {
+    my $looked_up_fast;
+    if ($requested_tess_id) {
+        # try using the last method first
+        $looked_up_fast = lookup_skycells_fast($ipprc, \@lines, $requested_tess_id, $ra, $dec, $verbose);
+    } 
+    
+    if (!$looked_up_fast) {
         my $command = "$whichimage $ra $dec";
         $command .= " --tess_id $requested_tess_id" if $requested_tess_id;
@@ -1302 +1446 @@
     $image->{astrom} = $astromFile;
     $image->{cam_path_base} = $camRoot;
+    $image->{cam_id} = $camRun->{cam_id};
 
     return 1;
@@ -1649 +1794 @@
 sub filterRuns {
     my $stage      = shift;
+    my $drop_duplicate_runs = shift;
     my $need_magic = shift;
     my $inputs     = shift;
@@ -1694 +1840 @@
         }
 
-        $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
+        # $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
 
         # skip if we need magicked run and this one has never been magicked
@@ -1702 +1848 @@
         }
 
-        if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) {
+        if (($exp_id == $last_exp_id) and ($drop_duplicate_runs || ($run_id != $last_run_id))) {
             print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed;
             next;
@@ -1873 +2019 @@
     return $diff_mode;
 }
+
+sub setErrorCodesForRows {
+    my $rowList = shift;
+    my $error_code = shift;
+
+    foreach my $row (@$rowList) {
+        $row->{error_code} = $error_code;
+    }
+
+}
+
+# Convert from skycell_id to projection_cell
+# This code is not particularly elegant but I think that it matches the SQL used
+# to create stackAssocations does. 
+
+sub skycell_id_to_projection_cell {
+    my $skycell_id = shift;
+
+    # split skycell_id into '.' separated components
+    my @strs = split '\.', $skycell_id;
+
+    my $num = scalar @strs;
+
+    # projection_cell is 'skycell_id' plus all .num values except the last one
+    my $projection_cell = $strs[0];
+    for (my $i=1; $i < $num - 1; $i++) {
+        $projection_cell .= ".$strs[$i]";
+    }
+
+    return $projection_cell;
+}
+
+# lookup_skycells_fast() 
+# Lookup using a "skycell server" process which is implemented by running dvoImagesAtCoords in "server" mode.
+# Repeated calls to this function will reuse an existing process. This avoids having to loading the
+# tessellation over and over again for each skycell lookup.
+# When this parse process exists, the pipes are closed which causes the skycell server to exit.
+
+my $scs_tess_id;    # tess_id for existing scs (skycell server) (if any)
+my $scs_out;        # reference filehandle for sending data to the scs
+my $scs_in;         # filehandle for response from scs
+
+sub lookup_skycells_fast {
+    my ($ipprc, $results, $tess_id, $ra, $dec, $verbose) = @_;
+
+    # see if we need to start the scs (skycell server)
+
+    my $very_verbose = 0;
+
+    if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) {
+        if ($scs_tess_id) {
+            # tess_id has changed, close down existing scs
+            close $scs_out;
+            $scs_out = undef;
+            close $scs_in;
+            $scs_in = undef;
+            $scs_tess_id = undef;
+        }
+
+        # convert tess_id to a directory name
+        my $tess_dir = $ipprc->tessellation_catdir( $tess_id );
+        unless ($tess_dir) {
+            print STDERR "Unrecognized tess_id: $tess_id\n";
+            return 0;
+        }
+
+        # convert tess_dir to an absolute directory name
+        my $tess_dir_resolved = $ipprc->convert_filename_absolute( $tess_dir );
+        unless ($tess_dir_resolved and -d $tess_dir_resolved) {
+            print STDERR "failed to resolve tessellation directory for $tess_id\n";
+            return 0;
+        }
+
+        # start an scs
+        print "Starting skycell server for $tess_dir_resolved.\n";
+
+        # create pipes for communicating with the server
+        pipe $scs_in, SERVER_WRITER;
+        pipe SERVER_READER, $scs_out;
+
+        # set our output to be unbuffered
+        $scs_out->autoflush(1);
+
+        # FORK
+        my $scs_pid = fork();
+
+        if (!$scs_pid) {
+            unless (defined $scs_pid) {
+                # This is still in parent process. No joy.
+                print STDERR "fork of skycell server failed: $!";
+                return 0;
+            }
+
+            # This code is running in the child process - the skycell server
+            # Close file handles for the parent's ends of the pipes
+            close $scs_in;
+            close $scs_out;
+            # close stdio filehandles. we are about to redirect them
+            # (STDERR stays open pointing to the parser log)
+            close STDIN;
+            close STDOUT;
+
+            # redirect our stdio file handles to the proper end of the pipes
+            open (STDIN,  "<&SERVER_READER")   or die "\nSCS: failed to redirect stdin";
+            open (STDOUT, ">>&SERVER_WRITER")  or die "\nSCS: failed to redirect stdout";
+
+            # All set. Now exec dvoImagesAtCoords
+
+            my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -";
+
+            print STDERR "SCS: execing $command\n" if $very_verbose;
+
+            unless(exec $command) {
+                # note parent will notice that we are gone due to the pipes getting broken
+                # when this child dies.
+                die "SCS: failed to exec $command";
+            }
+        }
+
+        # This code is running in the parent process (the parser).
+        # We have succesfully launched the scs process.
+        # Close file handles for the scs' end of the pipes.
+        close SERVER_WRITER;
+        close SERVER_READER;
+
+        # remember the tess_id 
+        $scs_tess_id = $tess_id;
+    }
+
+    # send coordinates to the skycell server and wait for the results 
+    # (which come nearly instantly which is the point of doing this)
+
+    # If pipes to the skycell server die, don't abort. 
+    # Our reads and writes detect errors and act sensibly.
+    local $SIG{PIPE} = 'IGNORE';
+
+    # write the coordinates to the pipe
+    my $write_ok = print $scs_out "1 $ra $dec\n";
+    if (!$write_ok) {
+        # server process probably didn't start properly or died. Fail.
+        # Caller will attempt to use the conventional method.
+        print STDERR "write to skycell server failed. Error is $!\n";
+        return 0;
+    }
+
+    print STDERR "  sent coordinates to skycell server\n" if $very_verbose;
+
+    my $received_done = 0;
+
+    # Wait for response. If pipe breaks the read returns with no data.
+    while (my $line = <$scs_in>) {
+        chomp $line;
+        if ($line eq 'DONE') {
+            # No more results for these coordinates.
+            $received_done = 1;
+            print STDERR "    received DONE\n" if $very_verbose;
+            last;
+        }
+        print STDERR "    received $line\n" if $very_verbose;
+
+        # The caller expects the output to be in the format used by whichimage
+        # which omits the point number and includes the tess_id.
+        my ($ptnum, $raout, $decout, $skycell_id) = split " ", $line;
+
+        my $out = "$raout $decout $tess_id $skycell_id";
+
+        # print the result to the log
+        print "$out\n" if $verbose;
+
+        push @$results, $out;
+    }
+
+    print STDERR "Out of wait for responses loop\n" if $very_verbose;
+
+    if (!$received_done) {
+        # something has gone wrong.
+        print STDERR "Read loop exited without receiving DONE.\n";
+
+        die "BAILING out" if $very_verbose;
+
+        # forget about the existing server instance
+        $scs_tess_id = undef;
+        close $scs_out;
+        close $scs_in;
+
+        # drop any results received so far
+        @$results = ();
+        return 0;
+    }
+
+    return 1;
+}
+
         
 

branches/eam_branches/ps2-tc3-20130727/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -29 +29 @@
                     $PSTAMP_SELECT_BACKMDL
                     $PSTAMP_SELECT_JPEG
+                    $PSTAMP_SELECT_EXP
+                    $PSTAMP_SELECT_NUM
                     $PSTAMP_SELECT_UNCOMPRESSED
                     $PSTAMP_SELECT_INVERSE
@@ -35 +37 @@
                     $PSTAMP_USE_IMFILE_ID
                     $PSTAMP_NO_WAIT_FOR_UPDATE
-                    $PSTAMP_REQUEST_UNCENSORED
-                    $PSTAMP_REQUIRE_UNCENSORED
+                    $PSTAMP_SELECT_EXPJPEG
+                    $PSTAMP_SELECT_NUMJPEG
                     $PSTAMP_SUCCESS
                     $PSTAMP_FIRST_ERROR_CODE
@@ -53 +55 @@
                     $PSTAMP_NO_OVERLAP
                     $PSTAMP_NOT_AUTHORIZED
+                    $PSTAMP_NO_VALID_PIXELS
                     );
 our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
@@ -71 +74 @@
 our $PSTAMP_SELECT_BACKMDL   = 32;
 our $PSTAMP_SELECT_JPEG      = 64;
-# unused 128
-# unused 256
+our $PSTAMP_SELECT_EXP       = 128;
+our $PSTAMP_SELECT_NUM       = 256;
 our $PSTAMP_SELECT_UNCOMPRESSED = 512;
 our $PSTAMP_SELECT_INVERSE      = 1024;
@@ -82 +85 @@
 our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
 
-# these bits will be repurposed
-our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
-our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
+our $PSTAMP_SELECT_EXPJPEG     = 0x10000;
+our $PSTAMP_SELECT_NUMJPEG     = 0x20000;
+
+# these bits have been repurposed. They were only exposed to MOPS and IFA and they have adapted.
+#our $PSTAMP_REQUEST_UNCENSORED = 0x10000;
+#our $PSTAMP_REQUIRE_UNCENSORED = 0x20000;
 
 # job and result codes
@@ -97 +103 @@
 our $PSTAMP_INVALID_REQUEST  = 21;
 our $PSTAMP_UNKNOWN_PROJECT  = 22;
-our $PSTAMP_UNKNOWN_PRODUCT  = 22;  #this error code was mis-named it is left for compatabiliyt
+our $PSTAMP_UNKNOWN_PRODUCT  = 22;  #this error code was a typo it is left for compatabiliy
 our $PSTAMP_NO_IMAGE_MATCH   = 23;
 our $PSTAMP_NOT_DESTREAKED   = 24;
 our $PSTAMP_NOT_AVAILABLE    = 25;
-our $PSTAMP_GONE             = 26;  # this value is used in ippTools
+our $PSTAMP_GONE             = 26;  # this value is also used in ippTools
 our $PSTAMP_NO_JOBS_QUEUED   = 27;
 our $PSTAMP_NO_OVERLAP       = 28;
 our $PSTAMP_NOT_AUTHORIZED   = 29;
+our $PSTAMP_NO_VALID_PIXELS  = 30;
 
 
@@ -143 +150 @@
 PSTAMP_NO_OVERLAP
 PSTAMP_NOT_AUTHORIZED
-PSTAMP_NOT_AUTHORIZED
+PSTAMP_NO_VALID_PIXELS
 );