Changeset 36057

Timestamp:

Aug 30, 2013, 1:00:26 PM (13 years ago)

Author:

bills

Message:

If request specifies the tess_id set up a "skycell server" process and
and send lookups to it. This speeds up parsing of bycoord requests by about
a factor of three by avoiding repeated invocations of dvoImagesAtCoords each
of which needs to read the entire tessellation directory.

Location:

trunk

Files:

• Ohana/src/getstar/src/dvoImagesAtCoords.c (modified) (3 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (3 diffs)

trunk/Ohana/src/getstar/src/dvoImagesAtCoords.c

@@ -9 +9 @@
 static int makePoint(double ra, double dec, Point **pointsOut);
 static int ListImagesAtCoords (Image *dbImages, Point *points, int Npoints);
+static int readPointFromFile(FILE *f, Point *pt);
 
 int main (int argc, char **argv) {
@@ -23 +24 @@
   
   Point *points;
+  int readStdin = 0;
   if (coordsFile) {
-      Npoints = readPoints(coordsFile, &points);
+      if (strcmp(coordsFile, "-") == 0) {
+        readStdin = 1;
+      } else {
+        Npoints = readPoints(coordsFile, &points);
+      }
   } else {
       Npoints = makePoint(cmd_line_ra, cmd_line_dec, &points);
   }
-  if (!Npoints) {
+  if (!Npoints && !readStdin) {
     exit(1);
   }
@@ -61 +67 @@
   }
   
-  if (MatchCoords (dbImages, NdbImages, points, Npoints)) {
-    ListImagesAtCoords(dbImages, points, Npoints);
-    exit(0);
+  if (readStdin) {
+    Point pt;
+    memset(&pt, 0, sizeof(pt));
+    pt.Nmatches = 0;
+    ALLOCATE(pt.matches, Match, 20);
+    pt.arrayLength = 20;
+
+    int status;
+    while ((status = readPointFromFile(stdin, &pt))) {
+      if (status < 0) {
+        fprintf(stdout, "ERROR malformed input line\n");
+        exit(1);
+      }
+      if (MatchCoords (dbImages, NdbImages, &pt, 1)) {
+        ListImagesAtCoords(dbImages, &pt, 1);
+      }
+      fprintf(stdout, "DONE\n");
+      fflush(stdout);
+      pt.Nmatches = 0;
+    }
   } else {
-    exit(PSTAMP_NO_OVERLAP);
-  }
+    if (MatchCoords (dbImages, NdbImages, points, Npoints)) {
+      ListImagesAtCoords(dbImages, points, Npoints);
+    } else {
+      exit(PSTAMP_NO_OVERLAP);
+    }
+  }
+  exit(0);
+}
+
+static int readPointFromFile(FILE *f, Point *pt) {
+    char buf[80];
+    char *good = fgets(buf, 80, f);
+    if (!good) {
+        return 0;
+    }
+    // fprintf(stderr, "READ: %s\n", buf);
+    int Nread = sscanf(buf, "%d %lf %lf\n", &pt->id, &pt->ra, &pt->dec);
+    if (Nread == 3) {
+        // got one
+        return 1;
+    } else if (Nread == -1) {
+        // all done time to go
+        return 0;
+    } else {
+        // fprintf(stderr, "malformed input line: %d\n", Nread);
+        return -1;
+    }
 }
 

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
 use POSIX;
 use Time::HiRes qw(gettimeofday);
+use IO::Handle;
 
 my $dvo_verbose = 0;
@@ -1288 +1289 @@
 
     my @lines;
-    {
+    my $looked_up_fast;
+    if ($requested_tess_id) {
+        # try using the last method first
+        $looked_up_fast = lookup_skycells_fast($ipprc, \@lines, $requested_tess_id, $ra, $dec, $verbose);
+    } 
+    
+    if (!$looked_up_fast) {
         my $command = "$whichimage $ra $dec";
         $command .= " --tess_id $requested_tess_id" if $requested_tess_id;
@@ -2028 +2035 @@
     return $projection_cell;
 }
+
+# lookup_skycells_fast() 
+# Lookup using a "skycell server" process which is implemented by running dvoImagesAtCoords in "server" mode.
+# Repeated calls to this function will reuse an existing process. This avoids having to loading the
+# tessellation over and over again for each skycell lookup.
+# When this parse process exists, the pipes are closed which causes the skycell server to exit.
+
+my $scs_tess_id;    # tess_id for existing scs (skycell server) (if any)
+my $scs_out;        # reference filehandle for sending data to the scs
+my $scs_in;         # filehandle for response from scs
+
+sub lookup_skycells_fast {
+    my ($ipprc, $results, $tess_id, $ra, $dec, $verbose) = @_;
+
+    # see if we need to start the scs (skycell server)
+
+    if (!$scs_tess_id or ($scs_tess_id ne $tess_id)) {
+        if ($scs_tess_id) {
+            # tess_id has changed, close down existing scs
+            close $scs_out;
+            $scs_out = undef;
+            close $scs_in;
+            $scs_in = undef;
+            $scs_tess_id = undef;
+        }
+
+        # convert tess_id to a directory name
+        my $tess_dir = $ipprc->tessellation_catdir( $tess_id );
+        unless ($tess_dir) {
+            print STDERR "Unrecognized tess_id: $tess_id\n";
+            return 0;
+        }
+
+        # convert tess_dir to an absolute directory name
+        my $tess_dir_resolved = $ipprc->convert_filename_absolute( $tess_dir );
+        unless ($tess_dir_resolved and -d $tess_dir_resolved) {
+            print STDERR "failed to resolve tessellation directory for $tess_id\n";
+            return 0;
+        }
+
+        # start an scs
+
+        # create pipes for communicating with the server
+        pipe $scs_in, SERVER_WRITER;
+        pipe SERVER_READER, $scs_out;
+
+        # set our output to be unbuffered
+        $scs_out->autoflush(1);
+
+        # FORK
+        my $scs_pid = fork();
+
+        if (!$scs_pid) {
+            unless (defined $scs_pid) {
+                # This is still in parent process. No joy.
+                print STDERR "fork of skycell server failed: $!";
+                return 0;
+            }
+
+            # This code is running in the child process - the skycell server
+            # Close file handles for the parent's ends of the pipes
+            close $scs_in;
+            close $scs_out;
+            # close stdio filehandles. we are about to redirect them
+            # (STDERR stays open pointing to the parser log)
+            close STDIN;
+            close STDOUT;
+
+            # redirect our stdio file handles to the proper end of the pipes
+            open (STDIN,  "<&SERVER_READER")   or die "\nSCS: failed to redirect stdin";
+            open (STDOUT, ">>&SERVER_WRITER")  or die "\nSCS: failed to redirect stdout";
+
+            # All set. Now exec dvoImagesAtCoords
+
+            my $command = "$dvoImagesAtCoords -D CATDIR $tess_dir_resolved -coords -";
+
+            print STDERR "SCS: execing $command\n";
+
+            unless(exec $command) {
+                # note parent will notice that we are gone due to the pipes getting broken
+                # when this child dies.
+                die "SCS: failed to exec $command";
+            }
+        }
+
+        # This code is running in the parent process (the parser).
+        # We have succesfully launched the scs process.
+        # Close file handles for the scs' end of the pipes.
+        close SERVER_WRITER;
+        close SERVER_READER;
+
+        # remember the tess_id 
+        $scs_tess_id = $tess_id;
+    }
+
+    # send coordinates to the skycell server and wait for the results 
+    # (which come nearly instantly which is the point of doing this)
+
+    # If pipes to the skycell server die, don't abort. 
+    # Our reads and writes detect errors and act sensibly.
+    local $SIG{PIPE} = 'IGNORE';
+
+    # write the coordinates to the pipe
+    my $write_ok = print $scs_out "1 $ra $dec\n";
+    if (!$write_ok) {
+        # server process probably didn't start properly or died. Fail.
+        # Caller will attempt to use the conventional method.
+        print STDERR "write to skycell server failed. Error is $!\n";
+        return 0;
+    }
+
+    my $very_verbose = 0;
+    print STDERR "  sent coordinates to skycell server\n" if $very_verbose;
+
+    my $received_done = 0;
+
+    # Wait for response. If pipe breaks the read returns with no data.
+    while (my $line = <$scs_in>) {
+        chomp $line;
+        if ($line eq 'DONE') {
+            # No more results for these coordinates.
+            $received_done = 1;
+            print STDERR "    received DONE\n" if $very_verbose;
+            last;
+        }
+        print STDERR "    received $line\n" if $very_verbose;
+
+        # The caller expects the output to be in the format used by whichimage
+        # which omits the point number and includes the tess_id.
+        my ($ptnum, $raout, $decout, $skycell_id) = split " ", $line;
+
+        my $out = "$raout $decout $tess_id $skycell_id";
+
+        # print the result to the log
+        print "$out\n" if $verbose;
+
+        push @$results, $out;
+    }
+
+    print STDERR "Out of wait for responses loop\n" if $very_verbose;
+
+    if (!$received_done) {
+        # something has gone wrong.
+        print STDERR "Read loop exited without receiving DONE.\n";
+
+        die "BAILING out" if $very_verbose;
+
+        # forget about the existing server instance
+        $scs_tess_id = undef;
+        close $scs_out;
+        close $scs_in;
+
+        # drop any results received so far
+        @$results = ();
+        return 0;
+    }
+
+    return 1;
+}
+