Changeset 36057 for trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

Timestamp:

Aug 30, 2013, 1:00:26 PM (13 years ago)

Author:

bills

Message:

If request specifies the tess_id set up a "skycell server" process and
and send lookups to it. This speeds up parsing of bycoord requests by about
a factor of three by avoiding repeated invocations of dvoImagesAtCoords each
of which needs to read the entire tessellation directory.

File:

• trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (3 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
 use POSIX;
 use Time::HiRes qw(gettimeofday);
+use IO::Handle;
 
 my $dvo_verbose = 0;
@@ -1288 +1289 @@
 
     my @lines;
-    {
+    my $looked_up_fast;
+    if ($requested_tess_id) {
+        # try using the last method first
+        $looked_up_fast = lookup_skycells_fast($ipprc, \@lines, $requested_tess_id, $ra, $dec, $verbose);
+    } 
+    
+    if (!$looked_up_fast) {
         my $command = "$whichimage $ra $dec";
         $command .= " --tess_id $requested_tess_id" if $requested_tess_id;
@@ -2028 +2035 @@
     return $projection_cell;
 }
+
+# lookup_skycells_fast() 
+# Lookup using a "skycell server" process which is implemented by running dvoImagesAtCoords in "server" mode.
+# Repeated calls to this function will reuse an existing process. This avoids having to loading the
+# tessellation over and over again for each skycell lookup.
+# When this parse process exists, the pipes are closed which causes the skycell server to exit.
+
+my $scs_tess_id;    # tess_id for existing scs (skycell server) (if any)
+my $scs_out;        # reference filehandle for sending data to the scs
+my $scs_in;         # filehandle for response from scs
+
+sub lookup_skycells_fast {
+    my ($ipprc, $results, $tess_id, $ra, $dec, $verbose) = @_;
+
+    # see if we need to start the scs (skycell server)
+
+    if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) {
+        if ($scs_tess_id) {
+            # tess_id has changed, close down existing scs
+            close $scs_out;
+            $scs_out = undef;
+            close $scs_in;
+            $scs_in = undef;
+            $scs_tess_id = undef;
+        }
+
+        # convert tess_id to a directory name
+        my $tess_dir = $ipprc->tessellation_catdir( $tess_id );
+        unless ($tess_dir) {
+            print STDERR "Unrecognized tess_id: $tess_id\n";
+            return 0;
+        }
+
+        # convert tess_dir to an absolute directory name
+        my $tess_dir_resolved = $ipprc->convert_filename_absolute( $tess_dir );
+        unless ($tess_dir_resolved and -d $tess_dir_resolved) {
+            print STDERR "failed to resolve tessellation directory for $tess_id\n";
+            return 0;
+        }
+
+        # start an scs
+
+        # create pipes for communicating with the server
+        pipe $scs_in, SERVER_WRITER;
+        pipe SERVER_READER, $scs_out;
+
+        # set our output to be unbuffered
+        $scs_out->autoflush(1);
+
+        # FORK
+        my $scs_pid = fork();
+
+        if (!$scs_pid) {
+            unless (defined $scs_pid) {
+                # This is still in parent process. No joy.
+                print STDERR "fork of skycell server failed: $!";
+                return 0;
+            }
+
+            # This code is running in the child process - the skycell server
+            # Close file handles for the parent's ends of the pipes
+            close $scs_in;
+            close $scs_out;
+            # close stdio filehandles. we are about to redirect them
+            # (STDERR stays open pointing to the parser log)
+            close STDIN;
+            close STDOUT;
+
+            # redirect our stdio file handles to the proper end of the pipes
+            open (STDIN,  "<&SERVER_READER")   or die "\nSCS: failed to redirect stdin";
+            open (STDOUT, ">>&SERVER_WRITER")  or die "\nSCS: failed to redirect stdout";
+
+            # All set. Now exec dvoImagesAtCoords
+
+            my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -";
+
+            print STDERR "SCS: execing $command\n";
+
+            unless(exec $command) {
+                # note parent will notice that we are gone due to the pipes getting broken
+                # when this child dies.
+                die "SCS: failed to exec $command";
+            }
+        }
+
+        # This code is running in the parent process (the parser).
+        # We have succesfully launched the scs process.
+        # Close file handles for the scs' end of the pipes.
+        close SERVER_WRITER;
+        close SERVER_READER;
+
+        # remember the tess_id 
+        $scs_tess_id = $tess_id;
+    }
+
+    # send coordinates to the skycell server and wait for the results 
+    # (which come nearly instantly which is the point of doing this)
+
+    # If pipes to the skycell server die, don't abort. 
+    # Our reads and writes detect errors and act sensibly.
+    local $SIG{PIPE} = 'IGNORE';
+
+    # write the coordinates to the pipe
+    my $write_ok = print $scs_out "1 $ra $dec\n";
+    if (!$write_ok) {
+        # server process probably didn't start properly or died. Fail.
+        # Caller will attempt to use the conventional method.
+        print STDERR "write to skycell server failed. Error is $!\n";
+        return 0;
+    }
+
+    my $very_verbose = 0;
+    print STDERR "  sent coordinates to skycell server\n" if $very_verbose;
+
+    my $received_done = 0;
+
+    # Wait for response. If pipe breaks the read returns with no data.
+    while (my $line = <$scs_in>) {
+        chomp $line;
+        if ($line eq 'DONE') {
+            # No more results for these coordinates.
+            $received_done = 1;
+            print STDERR "    received DONE\n" if $very_verbose;
+            last;
+        }
+        print STDERR "    received $line\n" if $very_verbose;
+
+        # The caller expects the output to be in the format used by whichimage
+        # which omits the point number and includes the tess_id.
+        my ($ptnum, $raout, $decout, $skycell_id) = split " ", $line;
+
+        my $out = "$raout $decout $tess_id $skycell_id";
+
+        # print the result to the log
+        print "$out\n" if $verbose;
+
+        push @$results, $out;
+    }
+
+    print STDERR "Out of wait for responses loop\n" if $very_verbose;
+
+    if (!$received_done) {
+        # something has gone wrong.
+        print STDERR "Read loop exited without receiving DONE.\n";
+
+        die "BAILING out" if $very_verbose;
+
+        # forget about the existing server instance
+        $scs_tess_id = undef;
+        close $scs_out;
+        close $scs_in;
+
+        # drop any results received so far
+        @$results = ();
+        return 0;
+    }
+
+    return 1;
+}
+