Changeset 28017 for branches/czw_branch/20100427/pstamp

Timestamp:

May 18, 2010, 5:06:01 PM (16 years ago)

Author:

watersc1

Message:

Almost working version of the mask-stats/software revision code.

Almost.

Location:

branches/czw_branch/20100427

Files:

• . (modified) (1 prop)
• pstamp/doc/dsinstall.txt (copied) (copied from trunk/pstamp/doc/dsinstall.txt )
• pstamp/scripts/Makefile.am (modified) (2 diffs)
• pstamp/scripts/detectability_respond.pl (modified) (20 diffs)
• pstamp/scripts/dquery_finish.pl (modified) (7 diffs)
• pstamp/scripts/dqueryparse.pl (modified) (8 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (24 diffs)
• pstamp/scripts/pstamp_cleanup.pl (copied) (copied from trunk/pstamp/scripts/pstamp_cleanup.pl )
• pstamp/scripts/pstamp_dorequest.pl (modified) (1 diff)
• pstamp/scripts/pstamp_finish.pl (modified) (19 diffs)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (9 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (12 diffs)
• pstamp/scripts/pstampparse.pl (modified) (32 diffs)
• pstamp/scripts/request_finish.pl (modified) (4 diffs)
• pstamp/src/ppstampMakeStamp.c (modified) (3 diffs)
• pstamp/src/ppstampMosaic.c (modified) (1 diff)
• pstamp/test/detect_query_create (modified) (3 diffs)

branches/czw_branch/20100427/pstamp/scripts/Makefile.am

@@ -3 +3 @@
 
 install_files = \
+        pstamp_cleanup.pl \
         pstamp_finish.pl \
         pstamp_insert_request.pl \
@@ -20 +21 @@
         pstamp_checkdependent.pl \
         request_finish.pl \
+        detectability_respond.pl \
         detect_query_read \
         detect_response_create \

branches/czw_branch/20100427/pstamp/scripts/detectability_respond.pl

@@ -41 +41 @@
 my $EXTVER = 1.0;
 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
-my ($req_id,$req_name,$product,$need_magic,$missing_tools);
-my ($request_file,$output,$dbname,$verbose,$save_temps);
+my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps);
 GetOptions(
-    'input=s'          =>     \$request_file,
+    'input=s'         =>      \$request_file,
     'output=s'        =>      \$output,
+    'workdir=s'       =>      \$workdir,
     'dbname=s'        =>      \$dbname,
+    'dbserver=s'      =>      \$dbserver,
     'verbose'         =>      \$verbose,
     'save-temps'      =>      \$save_temps,
@@ -55 +57 @@
            -exitval => 3,
     ) unless
-    defined $request_file and defined $output and defined $dbname;
+    defined $request_file and defined $output and defined $workdir and defined $dbname;
 
 my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1);
@@ -71 +73 @@
 
 my $ipprc = PS::IPP::Config->new();
-#my $tmp_dir = "/data/${host}.0/tmp/";
+
+if (!$dbserver) {
+    $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+# This is hardcoded in for the moment.
+$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver);
+my $imagedb = $project->{dbname};
+if (!$imagedb) {
+    carp("failed to find imagedb for project: $project");
+}
 
 #
 # Parse input request file using detect_query_read (as it's already written).
 #
-
 my $dqr_command = "$detect_query_read --input $request_file";
 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run(command => $dqr_command, verbose => $verbose);
 unless ($success) {
-    my_die("Unable to perform $dqr_command error code: $error_code",-1,-1,-1,-1,-1,-1);
+    # This is a problem, because I'm not sure how we handle a failure to read something.
+    # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+    # we can't send that response back.
+    die("Unable to perform $dqr_command error code: $error_code");
 }
 my %query = ();
@@ -119 +132 @@
 my %image_list_hash;
 for (my $i = 1; $i < $Nrows; $i++) {
-#    print "$i $Nrows $query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]\n";
-
+    # This could use the fact that locate_images now accepts position arrays, but
+    # I'll save that for after I get the majority of things working.
     my $image_set_tmp  = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0],
                                         $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],
@@ -126 +139 @@
                                         $query{CONTENT}{ROWNUM}[$i],$verbose);
     unless (%$image_set_tmp) {
-        my_die("No images were returned.",$query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-               $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
-               $query{HEADER}{STAGE}[0]);
-    }
-    print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
-    # This appends, assuming that if we get an image, we also get the identical psf/mask/weight/etc.
+        # No images were returned, so create a dummy entry that 
+        $image_list_hash{'no_image'}{IMAGE}    = 'no_image';
+        $image_list_hash{'no_image'}{PSF}      = 'no_psf';
+        $image_list_hash{'no_image'}{MASK}     = 'no_mask';
+        $image_list_hash{'no_image'}{WEIGHT}   = 'no_weight';
+        $image_list_hash{'no_image'}{CATALOG}  = 'no_catalog';
+        $image_list_hash{'no_image'}{CLASS_ID} = 'no_class';
+        $image_list_hash{'no_image'}{ERROR}    = $PSTAMP_NO_IMAGE_MATCH;
+        push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]";
+        push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i];
+        next;
+    }
+#     print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n";
+#     print "    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n";
+#     print "    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
+
+    # This indexes the results for identical images into the same hash.
     $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE}    = $image_set_tmp->{IMAGE};
     $image_list_hash{$image_set_tmp->{IMAGE}}{PSF}      = $image_set_tmp->{PSF};
@@ -138 +162 @@
     $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG}  = $image_set_tmp->{CATALOG};
     $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID};
+    $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR}    = $image_set_tmp->{ERROR};
     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES};
     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM};
 }
+
 my $i = 0;
-
 foreach my $k (keys %image_list_hash) {
+    # If we errored out on finding an image, we need to not try to run psphot here.
+    if ($image_list_hash{$k}{ERROR} != 0) {
+        next;
+    }
     # Write coordinates of the requested targets to a file.
-    my ($coordfile,$coordname) = tempfile("/tmp/detect.coords.$i.XXXX", 
+    my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 
                                             UNLINK => !$save_temps);
-    my ($targetfile,$targetname) = tempfile("/tmp/detect.targets.$i.XXXX", 
+    my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 
                                             UNLINK => !$save_temps);
 
@@ -153 +182 @@
         print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n";
     }
-
+#    print "$k\n";
     # Convert the sky coordinates to image coordinates with ppCoord.
     my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname";
@@ -171 +200 @@
     }
 
-    
+#     print "psphot $image_list_hash{$k}{PSF}\n";
     # Run psphotForced on the target list.
-    my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "/tmp/", CLEANUP => !$save_temps);
+    my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
     $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]";
     
@@ -185 +214 @@
         run(command => $psphot_cmd, verbose => $verbose);
     unless ($success) {
-        my_die("Unable to perform $psphot_cmd. Error_code: $error_code",
-               $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-               $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
-               $query{HEADER}{STAGE}[0],$error_code);
+        $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR;
     }
 }
@@ -196 +222 @@
 #
 my @rownums = ();
+my @out_errors = ();
 my @psphot_Npix = ();
 my @psphot_Qfact= ();
 my @psphot_flux = ();
+
 foreach my $k (keys %image_list_hash) {
-    my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
-    
-    my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
-
-    push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
-    push @psphot_Npix,    @{ $tmp_Npix };
-    push @psphot_Qfact,   @{ $tmp_Qfact };
-    push @psphot_flux,    @{ $tmp_flux };
+    if ($image_list_hash{$k}{ERROR} == 0) {
+        my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
+        
+        my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
+        
+        push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
+        push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Npix,    @{ $tmp_Npix };
+        push @psphot_Qfact,   @{ $tmp_Qfact };
+        push @psphot_flux,    @{ $tmp_flux };
+    }
+    else {
+        push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
+        push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Npix,    (map { 0 }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Qfact,   (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_flux,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
+    }   
 }
 
@@ -214 +252 @@
                     $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0],
                     $query{HEADER}{obscode}[0],
-                    \@rownums, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux);
-
+                    \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux);
+# print "Wrote response file $output\n";
 #
 # Add to datastore
 #
-# my $finish_command = "dquery_finish.pl --req_id $req_id --req_name $req_name --product $product";
-# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#     run(command => $finish_command, verbose => $verbose);
-# unless ($success) {
-#     warn ("Unable to perform $finish_command error code: $error_code");
-# }
-
+# Files are added to the datastore by dquery_finish.pl
 #
 # Cleanup
 #
 # Since everything is written to temporary files, there should be nothing to cleanup.
+
 #
 # Utilities
@@ -242 +275 @@
     my $index  = shift;
     my $verbose = shift;
-    
+
+    # This is the set of things that we need in order to run psphotForced
     my $option_mask |= 1;
     $option_mask |= $PSTAMP_SELECT_IMAGE;
@@ -252 +286 @@
     my $mjd_max = $mjd + 1;
 
+    # Construct a row list. 
+    my @rowList;
+    $rowList[0]->{CENTER_X} = $ra;
+    $rowList[0]->{CENTER_Y} = $dec;
+    $rowList[0]->{ID} = 1;
+    $rowList[0]->{STAGE} = $stage;
+    $rowList[0]->{COORD_MASK} = 0;
+
+    #    print "$stage\n";
     # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
-    my @images = locate_images($ipprc,$dbname,"bycoord",$stage,
-                               undef,undef,undef,$option_mask,$need_magic,
-                               $ra,$dec,$mjd_min,$mjd_max,$filter . ".00000",undef,$verbose);  
+    my @images = locate_images($ipprc,$imagedb,
+                                \@rowList,
+                               "bycoord",$stage,
+                               undef,undef,undef,
+                               $option_mask,$need_magic,
+                               # $ra,$dec,
+                               $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose);  
 
     my %image_info  = ();
@@ -270 +317 @@
             }
             elsif ($stage eq 'stack') {
-                # Stacks hide the exposure name very well, so I'm
-                # choosing the stack_id as the FPA_ID to match. This
-                # probably needs to be approved somehow.
+                # Stacks hide the exposure name very well, so we can only match against stage_id
                 if (${ $j }{stage_id} ne $FPA_ID) {
                     next;
@@ -285 +330 @@
             }
             # Debug prints of all the components of this image
-            foreach my $k (keys %{ $j }) {
-                print "$i $j $k ${ $j }{$k}\n";
-            }
-            
+#           foreach my $k (keys %{ $j }) {
+#               if ($k eq 'row_index') {
+#                   print "$i $j $k @{${ $j }{$k} }\n";
+#               }
+#               print "$i $j $k ${ $j }{$k}\n";
+#           }
+
+            # Check for existance of the images. Drawn mostly from pstampparse.pl
+            my $run_state = ${ $j }{state};
+            my $data_state = ${ $j }{data_state};
+            $data_state = $run_state if $stage eq 'stack';
+            my $fault = 0;
+            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+                ($run_state eq 'drop') or 
+                ($run_state eq 'error_cleaned') or 
+                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+                # image is gone and it's not coming back
+                $fault = $PSTAMP_GONE;
+            }
+            elsif  (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) {
+                if (($stage eq 'stack')||($stage eq 'diff')) {
+                    # updating stacks and diffs isn't implemented
+                    $fault = $PSTAMP_NOT_IMPLEMENTED;
+                }
+                if ($stage eq 'chip') {
+                    my $burntool_state = ${ $j }{burntool_state};
+                    if ($burntool_state and (abs($burntool_state) < 14)) {
+                        $fault = $PSTAMP_NOT_AVAILABLE;
+                    }
+                }
+                
+                if ($fault == 0) {
+                    # This bombs us out to dqueryparse, which will then flag a job for this run to be updated.
+                    my_die_for_update($data_state,$query{HEADER}{STAGE}[0],
+                                      ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id},
+                                      $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE);
+                }
+            }
+
             # This image matches, so we want to save the information into our output structure
             $image_info{ROWNUM} = $index;
@@ -295 +375 @@
             $image_info{MASK}   = ${ $j }{mask};
             $image_info{WEIGHT} = ${ $j }{weight};
+            $image_info{ERROR}  = $fault;
             $image_info{SKY_COORDINATES} = "$ra $dec";
             # To do sky->image coordinate transformations, we need to use the cmf/smf file. If 
@@ -405 +486 @@
     my $obscode = shift;
     my $rownum_ref = shift;
+    my $out_err_ref = shift;
     my $psphot_Npix_ref = shift;
     my $psphot_Qfact_ref = shift;
@@ -413 +495 @@
     my $columns = [
         # matching rownum from detectability original request
-        { name => 'ROWNUM',   type => '20A', writetype => TSTRING }, 
+        { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
+        # any errors that occurred during processing
+        { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
         # number of pixels used in hypothetical PSF for the query detection
         { name => 'DETECT_N', type => 'V',   writetype => TULONG },
@@ -455 +539 @@
     for (my $i = 0; $i < $numRows; $i++) {
         push @{$colData{'ROWNUM'}},      ${ $rownum_ref }[$i];
+        push @{$colData{'ERROR_CODE'}},  ${ $out_err_ref }[$i];
         push @{$colData{'DETECT_N'}},    ${ $psphot_Npix_ref }[$i];
         push @{$colData{'DETECT_F'}},    ${ $psphot_Qfact_ref }[$i];
@@ -529 +614 @@
 
 }
+
+sub my_die_for_update {
+    my $state = shift;
+    my $stage = shift;
+    my $stage_id = shift;
+    my $component = shift;
+    my $need_magic = shift;
+    my $imagedb = shift;
+    my $exit_code = shift;
+
+    print "$state $stage $stage_id $component $need_magic $imagedb\n";
+    print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n";
+    exit($exit_code);
+}

branches/czw_branch/20100427/pstamp/scripts/dquery_finish.pl

@@ -20 +20 @@
 use PS::IPP::Config qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $product, $out_dir, $dbname, $dbserver, $verbose, $save_temps );
+my ( $req_id, $req_name, $req_file, $product, $outdir, $dbname, $dbserver, $verbose, $save_temps );
 
 GetOptions(
@@ -27 +27 @@
            'req_file=s' => \$req_file,
            'product=s'  => \$product,
-           'out_dir=s'  => \$out_dir,
+           'outdir=s'   => \$outdir,
            'dbname=s'   => \$dbname,
            'dbserver=s' => \$dbserver,
@@ -41 +41 @@
 $err .= "--req_name is required\n" if !$req_name;
 $err .= "--product is required\n" if !$product;
-$err .= "--out_dir is required\n" if !$out_dir;
+$err .= "--outdir is required\n" if !$outdir;
 
 die "$err" if $err;
@@ -76 +76 @@
     die "product directory does not exist $prod_dir";
 }
-my $out_dir = "$prod_dir/$req_name";
+my $outdir = "$prod_dir/$req_name";
 }
-if (! -e $out_dir ) {
+if (! -e $outdir ) {
     # something must have gone wrong at the parse stage
-    print STDERR "output fileset directory $out_dir does not exist\n" if $verbose;
-    if (! mkdir $out_dir ) {
+    print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
+    if (! mkdir $outdir ) {
         stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
-        die "cannot create output directory $out_dir";
+        die "cannot create output directory $outdir";
     }
-} elsif (! -d $out_dir) {
+} elsif (! -d $outdir) {
     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
-    die "output fileset directory $out_dir exists but is not a directory";
+    die "output fileset directory $outdir exists but is not a directory";
 }
 
@@ -118 +118 @@
 
 # XXX: have the jobs produce the reglist as with postage stamp requests
-my ($REGLIST, $reg_list) = tempfile("$out_dir/reqlist.XXXX", UNLINK => !$save_temps);
+my ($REGLIST, $reg_list) = tempfile("$outdir/reqlist.XXXX", UNLINK => !$save_temps);
+
 foreach my $job (@jobs) {
     my $job_id = $job->{job_id};
     my $response_file = "response${job_id}.fits";
-    my $response_path = "$out_dir/$response_file";
+    my $response_path = "$outdir/$response_file";
 
     if (-e $response_path) {
         # the job generated a response file put it into the Data Store
         print $REGLIST "$response_file|||table|\n";
+
+        # do the same if we have an error file.  Should the parse data be uploaded as well?
+        my $err_file = "parse_error.txt";
+        if (-e "$outdir/$err_file") {
+            print $REGLIST "$err_file|||text|\n";
+        }
     } else  {
         print STDERR "detect_query response file for job $job_id not found\n" if $verbose;
@@ -136 +143 @@
 if (-s $reg_list) {
     my $command = "$dsreg --add $req_name --product $product --list $reg_list";
-    $command .= " --copy --datapath $out_dir";
+    $command .= " --copy --datapath $outdir";
     $command .= " --type MOPS_DETECTABILITY_RESPONSE";
+    $command .= " --ps0 $req_id";
     $command .= " --dbname $dbname" if $dbname;
 
@@ -158 +166 @@
     my $verbose = shift;
     
-    my $command = "$pstamptool -updatereq -req_id $req_id -state stop";
-    $command   .= " -fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
+    $command   .= " -set_fault $fault" if $fault;
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;

branches/czw_branch/20100427/pstamp/scripts/dqueryparse.pl

@@ -2 +2 @@
 #
 # parse a MOPS_DETCTABILITY_QUERY table and create a results file
-#
-# Note: this file is currently only a placeholder which creates a fake response file
-# and adds a completed job to the database
 #
 
 use strict;
 use warnings;
-
+use Carp;
 use Getopt::Long qw( GetOptions );
 use Pod::Usage qw( pod2usage );
 use IPC::Cmd 0.36 qw( can_run run );
 
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
                        $PS_EXIT_UNKNOWN_ERROR
@@ -26 +25 @@
                        );
 
-my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
-
+my ($no_update, $imagedb, $label);
+my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
+my ($job_id,$rownum); # stuff from the post-update world
 #
 # parse args
@@ -35 +35 @@
         'file=s'          =>      \$req_file,
         'req_id=s'        =>      \$req_id,
-        'out_dir=s'       =>      \$out_dir,
+        'job_id=s'        =>      \$job_id,
+        'rownum=s'        =>      \$rownum,
+        'outdir=s'       =>      \$outdir,
+        'label=s'         =>      \$label,
         'product=s'       =>      \$product,
         'mode=s'          =>      \$mode,
@@ -44 +47 @@
 ) or pod2usage(2);
 
-my $err = "";
-
-if (!$req_file) {
-    $err .= "--file is required\n";
-}
-if (!$req_id) {
-    $err .= "--req_id is required\n";
-}
-if (!$out_dir) {
-    $err .="--out_dir is required\n";
-}
-if (!$product) {
-    $err .="--product is required\n";
-}
-
-
-die $err if ($err);
-
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
+die "--file or --job_id is required" if !$req_file;
+
+if ($mode ne "list_uri") {
+    die "req_id is required" if !$req_id;
+    die "outdir is required" if !$outdir;
+    die "product is required" if !$product;
+}
 my $missing_tools;
 my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1);
-my $fakedresponse = can_run('fakedresponse.pl') or (warn "Can't find fakedresponse.pl" and $missing_tools =1);
+my $detectresponse = can_run('detectability_respond.pl') or 
+    (warn "Can't find detectability_respond.pl" and $missing_tools = 1);
 my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1);
 
@@ -77 +71 @@
 }
 
+# just deal with these arguments once and for all
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+$detectresponse .= " --dbname $dbname" if $dbname;
+
+$no_update = 1 if $mode eq "list_job";
+
+# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing.
+if (!$job_id) {
+    my $argslist = "$outdir/parse.args";
+    open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n";
+    close ARGSLIST;
+}
+
 # get the query id and check the extname and version from the header
-my $fields_output;
+my $fields_output; 
 {
     my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
@@ -84 +93 @@
         run(command => $command, verbose => $verbose);
     
-#   fields doesn't return zero when it succeeds
-#    unless ($success) {
-#        print STDERR @$stderr_buf;
-#    }
     $fields_output = join "", @$stdout_buf;
 }
 my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
 
-die "$req_file is missing one of EXTNAME EXTVER or QUERY_ID" 
+my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
     if !(defined($extname) and defined($extver) and defined($req_name));
-
-die "$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table"
-                if $extname ne         "MOPS_DETECTABILITY_QUERY";
-die "$req_file is version $extver expecting 1" if $extver ne 1;
-
-if (0) {
-$out_dir .= "/$req_name";
-}
-if (! -e $out_dir ) {
-    mkdir $out_dir or die "cannot create output directory $out_dir";
-} elsif (! -d $out_dir ) {
-    die "output fileset directory $out_dir exists but is not a directory";
-}
-
-#
-# XXX Eventually we will parse the file here, to look up the list of input images to be processed
-# and queue a job for each image.
-# The request file will be one of the arguments. Each job will look at all rows and create entries
-# in the response file the coordinates that overlap the image.
-#
-# In the meantime we don't parse the file here, pass it to the program fakedresponse.
-# It creates a response file with a line for each row in the request file
-
-my $response_file = "$out_dir/response.fits";
+my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
+    if $extname ne "MOPS_DETECTABILITY_QUERY";
+my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR)
+    if $extver ne 1;
+
+# Set up the workdir for this query.
+if (! -e $outdir ) {
+    mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR);
+} elsif (! -d $outdir ) {
+    my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR);
+}
+
+
+# Pass along the request file to the response generator.
+my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
+my $data_to_update = '';
 {
-    my $command = "$fakedresponse --input $req_file --output $response_file --workdir $out_dir";
+    my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -127 +124 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        warn("Unable to perform $command error code: $error_code");
-    }
+        warn("Warn! Unable to perform $command error code: $error_code");
+    }
+    # Use the fault code to see if we can regenerate the missing data.
     $fault = $error_code >> 8;
-}
-
-my $job_id;
+    if ($fault == $PSTAMP_NOT_AVAILABLE) {
+        $data_to_update = (split /\n/, (join "", @$stdout_buf))[-1];    
+    }   
+}
+
 my $result;
-{
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
-    $command .= " -job_type detect_query -state stop -fault $fault";
+# If we returned correctly with a valid response file, get a job ID
+# for the completed work, and move the response to a standardized name.
+if ($fault == 0) {
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
+    $command .= " -job_type detect_query -state stop -fault 0";
     $command .= " -rownum 1";
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     if ($success) {
         $job_id = join "", @$stdout_buf;
         chomp $job_id;
         if ($job_id && -e $response_file) {
-            rename $response_file, "$out_dir/response${job_id}.fits";
+            rename $response_file, "$outdir/response${job_id}.fits";
         }
         $result = 0;
@@ -156 +155 @@
     }
 }
-
-
+else {
+    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
+    # Get the dependency id for the data we're requesting be updated.
+    my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
+
+    # Link this request to a job and link that job to any dependency
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
+    $command .= " -job_type detect_query -state run -fault 0";
+    $command .= " -rownum 1";
+    $command .= " -dep_id $dep_id" if $dep_id;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    if ($success) {
+        $job_id = join "", @$stdout_buf;
+        chomp $job_id;
+        if ($job_id && -e $response_file) {
+            # We shouldn't have a response file at this stage.
+            rename $response_file, "$outdir/response${job_id}.fits";
+        }
+        $result = 0;
+    } else {
+        warn("Unable to perform $command error code: $error_code");
+        $result = $error_code >> 8;
+    }
+}
+
+# This does not set the request state to stop.  That will happen with the request_finish.pl script, 
+# which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
 {
-    my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name -outProduct $product";
-    $command .= " -fault $result" if $result;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
+    $command .= " -set_fault $result" if $result;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
+        my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
     }
 }
  
 exit 0;
+
+
+# If we have to queue an update run, do so and create a new dependent
+sub queue_update_run {
+    my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_;
+
+    my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
+    
+    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
+        # We should have received one of these states, so if not, signal that we have a problem.
+        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+    }
+    my $dep_id;
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component ";
+    $command .= " -outdir $outdir";
+    $command .= " -need_magic" if $need_magic;
+
+    my $rlabel = "dq_ud_" . $label if $label;
+    $command .= " -rlabel $rlabel" if $rlabel;
+
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+        my $output = join "", @$stdout_buf;
+        chomp $output;
+        $dep_id = $output;
+
+        my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
+    } 
+    else {
+        print STDERR "skipping $command\n";
+        $dep_id = 42;
+    }
+
+    return($dep_id);
+}
+        
+    
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+    exit $fault;
+}

branches/czw_branch/20100427/pstamp/scripts/pstamp_checkdependent.pl

@@ -21 +21 @@
 use Carp;
 
+# XXX: put this in a module somewhere
+my $IPP_DIFF_MODE_WARP_WARP   = 1;
+my $IPP_DIFF_MODE_WARP_STACK  = 2;
+my $IPP_DIFF_MODE_STACK_WARP  = 3;
+my $IPP_DIFF_MODE_STACK_STACK = 4;
 
 my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic);
@@ -30 +35 @@
     'stage_id=i'    =>  \$stage_id,
     'component=s'   =>  \$component,
-    'imagedb=s'     =>  \$imagedb,      # dbname for images
+    'imagedb=s'     =>  \$imagedb,      # dbname for images.
     'rlabel=s'      =>  \$rlabel,
     'need_magic'    =>  \$need_magic,
@@ -56 +61 @@
 }
 
+# Append imagedb to the ippTools
 $chiptool    .= " -dbname $imagedb";
 $warptool    .= " -dbname $imagedb";
 $difftool    .= " -dbname $imagedb";
+$stacktool   .= " -dbname $imagedb";
 $magicdstool .= " -dbname $imagedb";
-
 
 my $ipprc = PS::IPP::Config->new();
@@ -67 +73 @@
     $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
 }
-
 
 
@@ -103 +108 @@
 # Got "it"
 
+my $magic_ok = 0;
+if ($stage eq 'diff') {
+    if ($it->{diff_mode} == $IPP_DIFF_MODE_STACK_STACK) {
+        # stack stack diffs don't need magic, but since the warps need to have the chips destreaked
+        # in order to be processed se set need_magic in the database for all runs
+        # Now the diffs themselves don't need to be destreaked so
+        $magic_ok = 1;
+    }
+}
 my $status = 0;
 if (($it->{state} eq 'full') or ($it->{state} eq 'update') and ($whole_run or ($it->{data_state} eq 'full'))
-        and (!$need_magic or $it->{magicked} > 0)) {
-
-    # This Dependency is satisfied. all done
+        and (!$need_magic or $magic_ok or $it->{magicked} > 0)) {
+
+    # This Dependency is satisfied. All done!
     my $command = "$pstamptool -updatedependent -set_state full -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
@@ -123 +137 @@
 } elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update') and 
         ($whole_run or ($it->{data_state} eq 'cleaned'))) {
+
+    # Update has not been queued for this component yet.  
 
     # XXX: if $whole_run this is going to repeatedly call -setimfiletoupdate. This will be a no-op
@@ -140 +156 @@
     }
 } else {
+    # update for this component has been queued by hasn't completed yet or the state is an error state
+    # XXX: in this state we aren't checking the dependents anymore. If something goes wrong this
+    # will never complete. Consider calling the 'queue_update' functions again just to check. They are relatively cheap
+
+
     print "${stage}Run $stage_id state is $it->{state} $component data_state is $it->{data_state}\n";
 
@@ -150 +171 @@
 
     # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl
-    # so runs in this state will not get queued
+    # so runs in this state should not get queued
     if ($stage eq 'chip') {
         my $burntool_state = $it->{burntool_state};
@@ -169 +190 @@
     }
     if (!$job_fault and ($stage eq 'chip')) {
-        $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $it->{raw_magicked}, $it->{dsRun_state});
+        # chip processing is done, start destreaking.
+        $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state});
     }
     if ($job_fault) {
@@ -186 +208 @@
     my $need_magic = shift; 
 
+
     my $dsRun_state;
     my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked
@@ -193 +216 @@
             $raw_all_magicked &= ($chip->{raw_magicked} > 0);
 
-            if ($chip->{data_state} ne 'update') {
+            if ($chip->{state} =~ /error/) {
+                faultJobs('stop', undef, undef, $PSTAMP_GONE);
+            } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
                 my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}";
                 $command .= " -set_label $rlabel" if $rlabel;
@@ -228 +253 @@
     }
 
-    my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $raw_all_magicked, $dsRun_state);
+    my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state);
 
     return $status;
@@ -247 +272 @@
 
     my $warp_id = $metadata->{warp_id};
+    my $state = $metadata->{state};
+    if ($state =~ /error/) {
+        print STDERR "warpRun $warp_id has state $state faulting jobs\n";
+        faultJobs('stop', undef, undef, $PSTAMP_GONE);
+        exit 0;
+    }
+
     if (!$whole_run) {
         my $skycell = $metadata;
@@ -256 +288 @@
             if !$data or scalar @$data == 0;
 
-        my $good_to_go = 1;
+        my $chips_ready = 1;
         my @chipsToUpdate;
         my $chip_id;
@@ -262 +294 @@
             $chip_id = $chip->{chip_id};
             if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
-                $good_to_go = 0;
+                $chips_ready = 0;
                 push @chipsToUpdate, $chip;
             } else {
@@ -269 +301 @@
         }
 
-        if ($good_to_go) {
+        if ($chips_ready) {
+            # the reason we defer setting the warp to update is so that we can handle error conditions at previous
+            # stages more easily.
             my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
             $command .= " -set_label $rlabel" if $rlabel;
@@ -333 +367 @@
     my $need_magic = shift; 
 
-# XXXX This is not ready to go yet XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXxx
-
     my $diff_id   = $metadata->{diff_id};
     my $diff_mode = $metadata->{diff_mode};
@@ -341 +373 @@
         my $skycell_id = $skycell->{skycell_id};
 
-        # put this in a module somewhere
-        my $IPP_DIFF_MODE_WARP_WARP   = 1;
-        my $IPP_DIFF_MODE_WARP_STACK  = 2;
-        my $IPP_DIFF_MODE_STACK_WARP  = 2;
-        my $IPP_DIFF_MODE_STACK_STACK = 4;
         if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
-            # check the state of the stack
-            my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}";
+            # check the state of the template stack
+            my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
             my $stack = runToolAndParseExpectOne($command, $verbose);
             my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
@@ -361 +388 @@
             # now check the warp
             $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
-            my $warp = runToolAndExpectOne($command, $verbose);
-            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
-                if !$warp;
-        }
-
-        my $good_to_go = 1;
-        my @chipsToUpdate;
-        my $chip_id;
-        my $warp_id;
-        my $data ;  #XXXX
-        foreach my $chip (@$data) {
-            $chip_id = $chip->{chip_id};
-            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
-                $good_to_go = 0;
-                push @chipsToUpdate, $chip;
-            } else {
-                # this chip is good to go
-            }
-        }
-
-        if ($good_to_go) {
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            my $warp = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
+
+            if ($warp->{data_state} ne 'full') {
+                return queue_update_warp($warp, 0, $rlabel, $need_magic);
+            }
+            # warps are ready fall through and queue the diff update
+        } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
+            my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
+            my $warp1 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
+
+            my $warps_ready = 1;
+            my $status = 0;
+            if ($warp1->{data_state} ne 'full') {
+                $warps_ready = 0;
+                $status = queue_update_warp($warp1, 0, $rlabel, $need_magic);
+                if ($status) {
+                    return $status;
                 }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif (scalar @chipsToUpdate > 0) {
-            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
-        }
-    } else {
-        my $warpRun = $metadata;
-        my $warp_id = $warpRun->{warp_id};
-        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
-        my $data = runToolAndParse($command, $verbose);
-        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
-            if !$data or scalar @$data != 1;
-
-        my $chipRun = $data->[0];
-
-        my $chipRunState = $chipRun->{state};
-        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
-            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
-            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
-            return 0;
-        }
-        my $warpRunState = $warpRun->{state};
-        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
-            # The inputs and outputs are ready. Queue the warpRun for update.
-
-            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
-                }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif ($chipRunState eq 'cleaned' or 
-            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
-            my $data = [$chipRun];
-            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
-        }
-    }
-
-    # return value may be the return status of script so zero is good
+            }
+            $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
+            my $warp2 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
+
+            if ($warp2->{data_state} ne 'full') {
+                $warps_ready = 0;
+                $status = queue_update_warp($warp2, 0, $rlabel, $need_magic);
+            }
+
+            if (!$warps_ready) {
+                # don't queue the diff update yet
+                return $status;
+            }
+
+        } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
+            # check the state of the input stack
+            my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
+            my $stack1 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
+
+            if ($stack1->{state} ne 'full') {
+                print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+                # this faults all jobs depending on this dep_id
+                faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
+                return $PSTAMP_GONE;
+            }
+            # check the state of the template stack
+            $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
+            my $stack2 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2;
+
+            if ($stack2->{state} ne 'full') {
+                print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+                # this faults all jobs depending on this dep_id
+                faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
+                return $PSTAMP_GONE;
+            }
+
+            # inputs are ready fall through and queue the diff update
+        } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) {
+            # check the state of the input stack
+            my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
+            my $stack = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
+
+            if ($stack->{state} ne 'full') {
+                print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+                # this faults all jobs depending on this dep_id
+                faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
+                return $PSTAMP_GONE;
+            }
+
+            # now check the template warp
+            $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
+            my $warp = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
+
+            if ($warp->{data_state} ne 'full') {
+                return queue_update_warp($warp, 0, $rlabel, $need_magic);
+            }
+            # warps are ready fall through and queue the diff update
+        } else {
+            my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR);
+        }
+
+        my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+    } else {
+        # XXX todo whole_run
+        return $PS_EXIT_PROG_ERROR;
+    }
+
+    # return value is a unix style exit status so zero is good
     return 0;
 }
@@ -530 +584 @@
     my ($state, $stage, $stage_id, $job_fault) = @_;
 
-    my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -549 +603 @@
     my $stage_id = shift;
     my $rlabel  = shift;
+    my $need_magic = shift;
     my $input_magicked = shift;
     my $dsRun_state = shift;
 
-    # XXX: this code is only fully coded and tested for chip stage
+    # XXX: this code assumes that destreaking is handled at the chip stage
     my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip';
 
@@ -578 +633 @@
                 print "skipping $command\n";
             }
+        } elsif ($dsRun_state eq 'failed_revert') {
+            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update";
+            faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE);
         } else {
             print "magicDSRun.state = $dsRun_state for chipRun $stage_id";
@@ -593 +651 @@
     carp $msg;
 
-    my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -600 +658 @@
                     run(command => $command, verbose => $verbose);
         unless ($success) {
-            my_die("failed to set pstampDependent.fault for dep_id: $dep_id",
-                $PS_EXIT_UNKNOWN_ERROR);
+            carp "$cmd failed";
         }
     } else {

branches/czw_branch/20100427/pstamp/scripts/pstamp_dorequest.pl

@@ -106 +106 @@
 {
     ## TODO: what about request status
-    my $command = "$pstamptool -updatereq -req_id $request_id -state stop"; 
+    my $command = "$pstamptool -updatereq -req_id $request_id -set_state stop"; 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbosity);

branches/czw_branch/20100427/pstamp/scripts/pstamp_finish.pl

@@ -25 +25 @@
 use PS::IPP::PStamp::Job qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
+my %imagedb_cache;
+
+my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
 
 # the char to the right of the bar may be used as a single - alias for the longer name
@@ -34 +36 @@
            'req_file=s'     => \$req_file,
            'product=s'      => \$product,
-           'out_dir=s'      => \$out_dir,
+           'outdir=s'       => \$outdir,
            'dbname=s'       => \$dbname,
            'dbserver=s'     => \$dbserver,
@@ -44 +46 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 
-die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n"
-    if !$req_id or !$req_name or !$req_file or !$product or !$out_dir;
+die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n"
+    if !$req_id or !$req_name or !$req_file or !$product or !$outdir;
+
+die "outdir is NULL\n" if $outdir eq "NULL";
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 if ($redirect_output) {
-    my $logDest = "$out_dir/psfinish.$req_id.log";
+    # XXX: what happens here if the directory does not exist? We check below
+    my $logDest = "$outdir/psfinish.$req_id.log";
     $ipprc->redirect_output($logDest);
 }
@@ -70 +75 @@
 
 if ($product eq "NULL") {
+    # nothing more to do
     stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
 }
-
-my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
-exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -82 +85 @@
     my $fileset = $req_name;
 
-    print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
-
-    if (!-e $out_dir) {
+    print STDERR "product: $product  REQ_NAME: $req_name $outdir\n" if $verbose;
+
+    if (!-e $outdir) {
         # something must have gone wrong parsing the request
-        print STDERR  "output directory $out_dir does not exist\n";
-
-        if (!mkdir $out_dir) {
-            print STDERR "cannot create output directory $out_dir";
+        print STDERR  "output directory $outdir does not exist\n";
+
+        if (!mkdir $outdir) {
+            print STDERR "cannot create output directory $outdir";
             stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
         }
 
 
-    } elsif (! -d $out_dir ) {
+    } elsif (! -d $outdir ) {
         # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
         # request
-        print STDERR "output directory $out_dir exists but is not a directory";
+        print STDERR "output directory $outdir exists but is not a directory";
         stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
@@ -116 +119 @@
     }
 
-    # at this point we need to find out what kind of request type it is and
-    # split the processing depending on the request type
-    # the only processing required for a detectabilty query is to build the output fileset.
-
-    # the following is for a postage stamp request
-
-    my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps);
+    # start building the list of files to be placed in the output fileset
+    my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps);
+
+    # results file
     print $rlf "results.fits|||table|\n";
+    # human readable representation of the results file
     print $rlf "results.mdc|||text|\n";
 
     my $err_file = "parse_error.txt";
-    if (-e "$out_dir/$err_file" ) {
+    if (-e "$outdir/$err_file" ) {
         print $rlf "$err_file|||text|\n";
     }
 
-    my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps);
-
+    # The results table definition file
+    my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
 
     # data for the header
     print $tdf "$req_name|$req_id|\n";
+
+    # get the list of jobs generated for this request
     my @jobs;
     {
@@ -151 +154 @@
                 print STDERR "Request $req_id produced no jobs.\n"
             }
-            # This should not happen. A fake job should be queued
+            # This should not happen. A fake job should have been entered
             stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
         } else {
+if (0) {
             my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
 
             my $jobs = parse_md_list($metadata);
+}
+            my $jobs = parse_md_fast($mdcParser, $output);
 
             @jobs = @$jobs;
@@ -163 +169 @@
 
     my $exp_info;
-    my $last_exp_id = -1;
     foreach my $job (@jobs) {
         my $job_id = $job->{job_id};
@@ -171 +176 @@
         my $exp_id = $job->{exp_id};
 
-
         if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
-            # this request had a duplicate request name. We can't put the results
-            # on the data store since the product name is already used
-            # maybe we could be more clever about this and choose a fileset name
-            stop_request_and_exit($req_id, $fault);
+            # this request had a duplicate request name yet the parser didn't give
+            # it an "ERROR style name.
+            stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
         }
         my ($row, $req_info, $project) = get_request_info($rows, $rownum);
@@ -198 +201 @@
             # get the metadata for the exposure (if any i.e. stack) 
             # returns an appropriate string if !$exp_id
-            $exp_info = get_exposure_info($image_db, $exp_id);
-            $last_exp_id = $exp_id;
+            $exp_info = get_exposure_info($job_params, $image_db, $exp_id);
         } else {
             my $filter = $job_params->{filter};
             $filter = "0" if !$filter;
             $exp_info = "0|0|0|$filter|0|0";
-            $last_exp_id = -1;
         }
 
         if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
-            my $jreglist = "$out_dir/reglist$job_id";
+            my $jreglist = "$outdir/reglist$job_id";
             if (open JRL, "<$jreglist") {;
                 # process the reglist file to get the list of files produced by this job
@@ -225 +226 @@
                     # If not found check the PHU. If that doesn't work just set them to zero.
                     # XXX do this more cleanly
-                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
                     if (!defined $ra_deg) {
-                        (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`;
+                        (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
                     }
                     $ra_deg = 0.0 if (!$ra_deg);
@@ -258 +259 @@
     # make the results file
     {
-        my $command = "$pstamp_results --input $table_def_name --output $out_dir/results.fits";
+        my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -266 +267 @@
         } else {
             # dump a textual representation
-            my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc";
+            my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
@@ -278 +279 @@
         # register the fileset
         my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
-        $command .= " --link --datapath $out_dir --ps0 $req_id";
-        $command .= " --dbname $dbname" if $dbname;
+        $command .= " --link --datapath $outdir --ps0 $req_id";
+# XXX: let dsreg and config handle resolving dbname and dbserver
+#        $command .= " --dbname $dbname" if $dbname;
 
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -292 +294 @@
     # set the request's state to stop
     {
-        my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault";
+        my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault";
         $command   .= " -dbname $dbname" if $dbname;
         $command   .= " -dbserver $dbserver" if $dbserver;
@@ -308 +310 @@
     my $fault  = shift;
 
-    my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault";
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -345 +347 @@
     $data_group = "null" if !$data_group;
 
-    # This is ugly, error prone and hard to change.
+    # XXX: This is ugly, error prone and hard to change.
     # Create a results file module and provide a list of the names (we have the data in the columns)
     my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
@@ -376 +378 @@
     my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc");
 
+    # no need to use parse_md_fast here
     my $results = parse_md_list($metadata);
     if (scalar @$results != 1) {
@@ -385 +388 @@
 
 sub get_exposure_info {
+    my $job_params = shift;
     my $image_db= shift;
     my $exp_id = shift;
 
-    if (!$exp_id or !$image_db) {
-        # no exposure id just return zeros
-        # XXX: we could put a value in for filter, but we don't have a good place to find it
-
-        #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
-        return "0|0|0|0|0|0";
-    }
-
-    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
-    my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id";
-
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        die "cannot get exposure information for $exp_id from image database $image_db";
-    }
-    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        print STDERR "no output returned from $command\n" if $verbose;
-        return undef;
-    }
-    my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
-
-    my $exposures = parse_md_list($metadata);
-    my $numExp = @$exposures;
-
-    die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1;
+    my ($dateobs, $ra, $decl, $filter, $exp_time, $exp_name);
+
+    $dateobs  = $job_params->{dateobs};
+    $ra       = $job_params->{ra};
+    $decl     = $job_params->{decl};
+    $filter   = $job_params->{filter};
+    $exp_time = $job_params->{exp_time};
+    $exp_name = $job_params->{exp_name};
+
+    unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name) {
+        # job params don't have all of the values that we need (most likely this is a diff stage job)
+        # go look up the exposure if we have one
+        if (!$exp_id or !$image_db) {
+            # no exposure id just return zeros
+            # XXX: we could put a value in for filter, but we don't have a good place to find it
+
+            #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
+            return "0|0|0|0|0|0";
+        }
+
+        my $exp;
+        # get cache of exposures for this image_db
+        my $exp_cache = $imagedb_cache{$image_db};
+        if (defined $exp_cache) {
+            $exp = $exp_cache->{$exp_id};
+        }
+
+        unless (defined $exp) {
+
+            my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+            if ($missing_tools) {
+                warn("Can't find required tools.");
+                exit ($PS_EXIT_CONFIG_ERROR);
+            }
+
+            my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id";
+
+            # run the tool and parse the output
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                # not sure if we should die here
+                die "cannot get exposure information for $exp_id from image database $image_db";
+            }
+            my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+            my $output = join "", @$stdout_buf;
+            if (!$output) {
+                print STDERR "no output returned from $command\n" if $verbose;
+                return undef;
+            }
+if (0) {
+            my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
+
+            my $exposures = parse_md_list($metadata);
+}
+            my $exposures = parse_md_fast($mdcParser, $output);
+            my $numExp = @$exposures;
+
+            die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1;
+
+            $exp = $exposures->[0];
+
+            unless (defined $exp_cache) {
+                my %new_exp_cache;
+                $exp_cache = \%new_exp_cache;
+                $imagedb_cache{$image_db} = $exp_cache;
+            }
+            $exp_cache->{$exp_id} = $exp;
+        } else {
+            print "found $exp_id in cache\n";
+        }
+        
+        #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
+        $dateobs  = $exp->{dateobs};
+        $ra       = $exp->{ra};
+        $decl     = $exp->{decl};
+        $filter   = $exp->{filter};
+        $exp_time = $exp->{exp_time};
+        $exp_name = $exp->{exp_name};
+    }
+
+    die "failed to find exp_info for $exp_id" unless (defined $dateobs and defined $ra and defined $decl and defined $filter and defined $exp_time and defined $exp_name);
     
-    my $exp = $exposures->[0];
-
-    #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
-
     use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0);
-    my $ra_deg   = $exp->{ra} * RADIANS_TO_DEGREES;
-    my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES;
-    my $mjd_obs = dateobs_to_mjd($exp->{dateobs});
-
-    my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}";
+    my $ra_deg   = $ra * RADIANS_TO_DEGREES;
+    my $decl_deg = $decl * RADIANS_TO_DEGREES;
+    my $mjd_obs = dateobs_to_mjd($dateobs);
+
+    my $info = "$mjd_obs|$ra_deg|$decl_deg|$filter|$exp_time|$exp_name";
             
     return $info;

branches/czw_branch/20100427/pstamp/scripts/pstamp_get_image_job.pl

@@ -87 +87 @@
 }
 
-my $out_dir = dirname($output_base);
+my $outdir = dirname($output_base);
 my $prefix = basename($output_base) . "_";
 my $results_file = $output_base . ".bundle_results";
@@ -102 +102 @@
     my $command = "$dist_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --component $component";
-    $command .= " --path_base $path_base --outdir $out_dir --results_file $results_file";
+    $command .= " --path_base $path_base --outdir $outdir --results_file $results_file";
     $command .= " --prefix $prefix";
     $command .= " --magicked" if $magicked;
@@ -139 +139 @@
 }
 
-my $reglist = "$out_dir/reglist$job_id";
+my $reglist = "$outdir/reglist$job_id";
 if (! open(REGLIST, ">$reglist") ) {
     my_die("failed to open registration list: $reglist", $PS_EXIT_UNKNOWN_ERROR);

branches/czw_branch/20100427/pstamp/scripts/pstamp_job_run.pl

@@ -74 +74 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -83 +84 @@
 my $jobStatus;
 if ($jobType eq "stamp") {
-    my $argslist = "$outputBase.args";
-    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
-    my $argString = <ARGSLIST>;
-    close ARGSLIST;
-    chomp $argString;
+    my $params = read_params_file($outputBase);
+
+    my $argString;
+    $argString = $params->{job_args};
 
     # XXX: should we do any other sanity checking?
-    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    # XXX: remove -astrom from argString and add it here
+
+    $argString .= " -file $params->{image}";
+    $argString .= " -mask $params->{mask}";
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        $argString .= " -variance $params->{weight}";
+    }
 
     my $command = "$ppstamp $outputBase $argString";
@@ -119 +127 @@
         my %extensions = ( $PSTAMP_SELECT_IMAGE  => "fits", 
                            $PSTAMP_SELECT_MASK   => "mk.fits",
-                           $PSTAMP_SELECT_WEIGHT => "wt.fits");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_WEIGHT);
+                           $PSTAMP_SELECT_VARIANCE => "wt.fits");
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE);
 
         foreach my $key (keys (%extensions)) {
@@ -139 +147 @@
         }
 
-        get_other_outputs($F, $outputBase, $options);
+        get_other_outputs($F, $outputBase, $options, $params);
 
         close $F;
@@ -165 +173 @@
     }
 } elsif ($jobType eq "detect_query") {
-    my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR);
+    # detect_query jobs are basically holders to note that we need a file updated before we can continue.
+    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
+    # run it again the same way.
+    my $argslist = "$outputBase/parse.args";
+    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $argString = <ARGSLIST>;
+    close ARGSLIST;
+    chomp $argString;
+
+    # XXX: should we do any other sanity checking?
+    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    my $command = "$dqueryparse --job_id $job_id $argString";
+    $command .= " --dbname $dbname" if $dbname;
+    $command .= " --dbserver $dbserver" if $dbserver;
+    $command .= " --verbose" if $verbose;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    # Although dqueryparse can potentially force more updates (and more jobs), it should still return success.
+    if ($success) {
+        $jobStatus = $PS_EXIT_SUCCESS;
+    } else {
+        $jobStatus = $error_code >> 8;
+        my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
+    }
 } else {
     my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
@@ -172 +205 @@
 # mark the job stopped in the database
 {
-    my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
-    $command .= " -fault $jobStatus" if $jobStatus;
+    my $command = "$pstamptool -updatejob -job_id $job_id -set_state stop"; 
+    $command .= " -set_fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -212 +245 @@
     my $output_base = shift;
     my $options = shift;
+    my $params = shift;
 
     if ($options & ( $PSTAMP_SELECT_CMF | $PSTAMP_SELECT_PSF | $PSTAMP_SELECT_BACKMDL)) {
-        my $comp = read_params_file($output_base);
-
-        my $stage = $comp->{stage};
+        if (!$params) {
+            $params = read_params_file($output_base);
+        }
+
+        my $stage = $params->{stage};
 
         # raw files don't have any other data products
@@ -225 +261 @@
         # detected in pstampparse so that the user can be notified with 
         # a message in parse_error.txt ("warp do not have a background model")
-        my $cmf_file = $comp->{cmf} if ($options & $PSTAMP_SELECT_CMF);
-        my $psf_file = $comp->{psf} if ($options & $PSTAMP_SELECT_PSF);
-        my $backmdl_file = $comp->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
-        my $pattern_file = $comp->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+        my $psf_file = $params->{psf} if ($options & $PSTAMP_SELECT_PSF);
+        my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
 
         my $outdir = dirname($output_base);
@@ -317 +353 @@
         my $command = "$pstamptool -updatejob";
         $command .= " -job_id $job_id";
-        $command .= " -fault $exit_code";
+        $command .= " -set_fault $exit_code";
         # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
-        $command .= " -state run";
+        $command .= " -set_state run";
         $command .= " -dbname $dbname" if defined $dbname;
         $command .= " -dbserver $dbserver" if defined $dbserver;

branches/czw_branch/20100427/pstamp/scripts/pstamp_parser_run.pl

@@ -28 +28 @@
 my $redirect_output;
 my $product;
+my $outdir;
 my $label;
 my $verbose;
@@ -37 +38 @@
     'uri=s'             =>  \$uri,
     'product=s'         =>  \$product,
+    'outdir=s'          =>  \$outdir,
     'label=s'           =>  \$label,
     'redirect-output'   =>  \$redirect_output,
@@ -81 +83 @@
 }
 
-# workdir is where all of the files generated for this request are placed
-# NOTE: this location needs to be kept in sync with the web interface ( request.php )
-my $datestr = strftime "%Y%m%d", gmtime;
-my $datedir = "$pstamp_workdir/$datestr";
-if (! -e $datedir ) {
-    mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+if (!$outdir or ($outdir eq "NULL")) {
+    # outdir is where all of the files generated for this request are placed
+    # NOTE: this location needs to be kept in sync with the web interface ( request.php )
+    my $datestr = strftime "%Y%m%d", gmtime;
+    my $datedir = "$pstamp_workdir/$datestr";
+    if (! -e $datedir ) {
+        mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+            $PS_EXIT_CONFIG_ERROR);
+    }
+
+    $outdir = "$datedir/$req_id";
+}
+
+if (! -e $outdir ) {
+    mkdir $outdir or my_die("failed to create working directory $outdir for request id $req_id", $req_id,
         $PS_EXIT_CONFIG_ERROR);
 }
-
-my $workdir = "$datedir/$req_id";
-if (! -e $workdir ) {
-    mkdir $workdir or my_die("failed to create working directory $workdir for request id $req_id", $req_id,
-        $PS_EXIT_CONFIG_ERROR);
-}
+    
 
 if ($redirect_output) {
-    my $logDest = "$workdir/psparse.$req_id.log";
+    my $logDest = "$outdir/psparse.$req_id.log";
     $ipprc->redirect_output($logDest);
 }
@@ -108 +114 @@
 
 my $fn = basename($uri);
-my $new_uri = "$workdir/$fn";
+my $new_uri = "$outdir/$fn";
 if ($uri =~ /^http:/) {
     # if the uri is an http uri download the file 
@@ -118 +124 @@
     }
 } elsif ($uri ne $new_uri) {
-    # put a link to the file into the workdir
+    # copy the request file into outdir
     if (-e $new_uri) {
+        # file exists already delete it incase the previous copy is bogus
         unlink $new_uri or my_die("failed to unlink $new_uri", $req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
     if (! copy $uri, $new_uri) {
-        my_die ("failed to copy request file $uri to workdir $workdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
+        my_die ("failed to copy request file $uri to workdir $outdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 }
@@ -152 +159 @@
         } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
             $reqType = 'dquery';
-            $parse_cmd = $dqueryparse;
+            $parse_cmd = "$dqueryparse";
+            $parse_cmd .= " --label $label" if $label;
         } else {
             print STDERR "Unknown request type $request_type found in $uri";
@@ -172 +180 @@
     # that they sent us a request file that we don't understand
 
-    my $command = "$pstamptool -updatereq -req_id $req_id -state run";
-    $command   .= " -reqType unknown";
-    $command   .= " -fault $PS_EXIT_DATA_ERROR";
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state run";
+    $command   .= " -set_reqType unknown";
+    $command   .= " -set_fault $PS_EXIT_DATA_ERROR";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -185 +193 @@
 }
 
-$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri";
+$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --outdir $outdir --file $uri";
 $parse_cmd .= " --dbname $dbname" if $dbname;
 $parse_cmd .= " --dbserver $dbserver" if $dbserver;
@@ -193 +201 @@
 my $fault;
 {
-    my $error_file_name = "$workdir/parse_error.txt";
+    my $error_file_name = "$outdir/parse_error.txt";
     # get rid of any error file from previous attempt to parse this request
     unlink $error_file_name if (-e $error_file_name);
@@ -218 +226 @@
     unless ($success) {
         $fault = $error_code >> 8;
-        $newState = "stop";
+#        $newState = "stop";
     }
 }
@@ -226 +234 @@
 #
 {
-    my $command = "$pstamptool -updatereq -req_id $req_id -state $newState";
-    $command   .= " -reqType $reqType" if $reqType;
-    $command   .= " -uri $new_uri" if $new_uri;
-    $command   .= " -fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state $newState";
+    $command   .= " -set_outdir $outdir";
+    $command   .= " -set_reqType $reqType" if $reqType;
+    $command   .= " -set_uri $new_uri" if $new_uri;
+    $command   .= " -set_fault $fault" if $fault;
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -268 +277 @@
     carp($msg);
 
-    my $command = "$pstamptool -updatereq -req_id $req_id  -fault $fault";
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;

branches/czw_branch/20100427/pstamp/scripts/pstampparse.pl

@@ -17 +17 @@
 use Carp;
 use POSIX;
-use DateTime;
+use Time::HiRes qw(gettimeofday);
 
 my $verbose;
@@ -25 +25 @@
 my $request_file_name;
 my $mode = "list_uri";
-my $out_dir;
+my $outdir;
 my $product;
 my $label;
@@ -34 +34 @@
     'file=s'    =>  \$request_file_name,
     'req_id=s'  =>  \$req_id,
-    'out_dir=s' =>  \$out_dir,
+    'outdir=s'  =>  \$outdir,
     'product=s' =>  \$product,
     'label=s'   =>  \$label,
@@ -50 +50 @@
 if ($mode ne "list_uri") {
     die "req_id is required"   if !$req_id;
-    die "out_dir is required"  if !$out_dir;
+    die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
 }
@@ -129 +129 @@
     # update the database with the request name. This will be used as the
     # the output data store's product name
-    my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
-    $command .= " -outProduct $product";
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_outProduct $product";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -147 +147 @@
 my $rows;
 {
+    my $start_request_file = gettimeofday();
     my $command = "$pstampdump $request_file_name";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -154 +155 @@
     }
     if (@$stdout_buf) {
-        my $table =  $mdcParser->parse(join "", @$stdout_buf) or
-            my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
-        $rows = parse_md_list($table);
-    }
+        $rows = parse_md_fast($mdcParser, join "", @$stdout_buf);
+    }
+    my $dtime_request_file = gettimeofday() - $start_request_file;
+    print "Time to read and parse request file: $dtime_request_file\n";
 
 }
@@ -223 +224 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
-        print STDERR "$rownum is not a valid ROWNUM\n"  if $verbose;
+        print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -229 +230 @@
     my $job_type = $row->{JOB_TYPE};
     if (($job_type ne "stamp") and ($job_type ne "get_image")) {
-        print STDERR "$job_type is not a valid JOB_TYPE\n"  if $verbose;
+        print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -237 +238 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
-        print STDERR "$req_type is not a valid REQ_TYPE\n"  if $verbose;
+        print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
     }
+    if ($job_type eq 'get_image') {
+        unless ($req_type eq 'byid' or $req_type eq 'byexp') {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' for JOB_TYPE 'get_image'\n";
+            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            return 0;
+        }
+    }
 
 
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        $row->{COMPONENT} = $component = "";
+        if ($job_type eq 'get_image') {
+            $row->{COMPONENT} = 'all';
+        } else {
+            $row->{COMPONENT} = $component = "";
+        }
     }
     $row->{TESS_ID} = "" if !defined $row->{TESS_ID};
@@ -258 +270 @@
     my $mjd_min = $row->{MJD_MIN};
     if (defined($mjd_min) and !validNumber($mjd_min)) {
-        print STDERR "$mjd_min is not a valid MJD_MIN\n"  if $verbose;
+        print STDERR "$mjd_min is not a valid MJD_MIN\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -264 +276 @@
     my $mjd_max = $row->{MJD_MAX};
     if (defined($mjd_max) and !validNumber($mjd_max)) {
-        print STDERR "$mjd_max is not a valid MJD_MAX\n"  if $verbose;
+        print STDERR "$mjd_max is not a valid MJD_MAX\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -288 +300 @@
 
     if (!$skycenter and !$component) {
-        print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose;
+        print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -295 +307 @@
     my $stage = $row->{IMG_TYPE};
     if (!check_image_type($stage)) {
-        print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose;
+        print STDERR "invalid IMG_TYPE for row $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -301 +313 @@
 
     if ((($job_type eq "stamp") or ($req_type eq "bycoord")) and ! validROI($row)) {
-        print STDERR "invalid ROI for row $rownum\n" if $verbose;
+        print STDERR "invalid ROI for row $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
         return 0;
@@ -307 +319 @@
 
     if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose;
+        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -319 +331 @@
     if ($req_type eq "bycoord") {
         if (!$skycenter) {
-            print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
+            print STDERR "center must be specified in sky coordintes for bycoord";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -327 +339 @@
     if (($req_type eq "byid") or ($req_type eq "bydiff")) {
         if (!validID($row->{ID})) {
-            print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose;
+            print STDERR "ID must be a positive integer for req_type $req_type\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0
@@ -349 +361 @@
     my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     if (!$proj_hash) {
-        print STDERR "project $project not found\n"  if $verbose;
+        print STDERR "project $project not found\n" ;
         foreach $row (@$rowList) {
             insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
@@ -383 +395 @@
         }
     }
+
+    $need_magic = 0 if $stage eq 'stack';
     
     my $numRows = scalar @$rowList;
-
-#    $tess_id = "" if !defined $tess_id;
-#    $component = "" if !defined $component;
 
     print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
@@ -393 +404 @@
     # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
     # request specification. An array reference is returned.
-    my $start_locate = DateTime->now->mjd;
+    my $start_locate = gettimeofday();
 
     # XXX: perhaps we should get rid of most of this argument list.
@@ -402 +413 @@
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
-    # XXX: why use mjd? It doesn't have great precision.
-    my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.;
+    my $dtime_locate = gettimeofday() - $start_locate;
     print "Time to locate_images for row $rownum $dtime_locate\n";
 
-    if (!$imageList or !@$imageList) {
-        print STDERR "no matching images found for row $rownum\n" if $verbose;
-        # note in this case queueJobs inserts the fake job for these rows
-    }
-    # handle this
+    # handle this XXX: what did I mean by this comment
     $row->{need_magic} = $need_magic;
 
@@ -502 +508 @@
     $image->{job_args} = $args;
 
-    # XXX: we can get rid of the following everything that we need is
-    # in the params file
-
-    $args .= " -file $imagefile";
-
-    if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-        $args .= " -mask $image->{mask}";
-    }
-    if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-        $args .= " -variance $image->{weight}";
-    }
-
     my $base = basename($image->{image});
     if (! $base =~ /.fits$/ ) {
@@ -520 +514 @@
     $base =~ s/.fits$//;
             
-    my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
-    my $argslist = "${output_base}.args";
-
-    # copy the argument list to a file
-    open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
-    print ARGSLIST "$args\n";
-    close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
-
+    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
     write_params($output_base, $image);
 
@@ -536 +523 @@
     # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
     if ($stage ne 'raw') {
-        # not ready to handle diff update yet. May never support stack
-        my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff'));
+        # updates for stack stage not supported yet
+        my $allow_wait_for_update = ($stage ne 'stack');
         my $run_state = $image->{state};
         my $data_state = $image->{data_state};
@@ -548 +535 @@
             $newState = 'stop';
             $fault = $PSTAMP_GONE;
-        } elsif ($allow_wait_for_update and 
-                (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) {
+        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
             if ($stage eq 'chip') {
                 my $burntool_state = $image->{burntool_state};
@@ -556 +542 @@
                     $fault = $PSTAMP_NOT_AVAILABLE;
                 }
+            } 
+            if (!$allow_wait_for_update) {
+                print STDERR "wait for update not supported for stage $stage yet\n";
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
             }
             if (!$fault) {
@@ -658 +649 @@
 }
 
-#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
 sub queueGetImageJobs
 {
@@ -706 +696 @@
         my $exp_id = $image->{exp_id};
             
-        my $output_base = "$out_dir/${rownum}_${job_num}";
+        my $output_base = "$outdir/${rownum}_${job_num}";
 
         write_params($output_base, $image);
@@ -731 +721 @@
                     # cause the image to be re-made
                     # set up to queue an update run
-                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, 
                         $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
                 }
@@ -878 +868 @@
     my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
+    # XXX: The update process for warp and subsequent stages requires 
+    # destreaking to run because the -pending queries require it when magicked > 0
+    # queries. 
+    # The case of stack-stack diffs is taken care of in pstamp_checkdependent
+    $need_magic = 1 if $imagedb eq 'gpc1';
+
     if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
         my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
@@ -883 +879 @@
 
     my $dep_id;
-    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component";
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
 

branches/czw_branch/20100427/pstamp/scripts/request_finish.pl

@@ -21 +21 @@
 use PS::IPP::Config qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output );
+my ( $req_id, $req_name, $req_file, $req_type, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output );
 
 GetOptions(
@@ -28 +28 @@
            'req_file=s' => \$req_file,
            'req_type=s' => \$req_type,
-           'out_dir=s'  => \$out_dir,
+           'outdir=s'  => \$outdir,
            'product=s'  => \$product,
            'dbname=s'   => \$dbname,
@@ -46 +46 @@
 $err .= "--req_name is required\n" if !$req_name;
 $err .= "--product is required\n" if !$product;
-# $err .= "--out_dir is required\n" if !$out_dir;
+# $err .= "--outdir is required\n" if !$outdir;
 
 die "$err" if $err;
 
 
-if (!$out_dir) {
-    $out_dir = dirname($req_file);
+if (!$outdir) {
+    $outdir = dirname($req_file);
 }
 
 if ($redirect_output) {
-    my $logDest = "$out_dir/reqfinish.$req_id.log";
+    my $logDest = "$outdir/reqfinish.$req_id.log";
     my $ipprc = PS::IPP::Config->new();
     $ipprc->redirect_output($logDest);
@@ -79 +79 @@
 }
 if ($finish_cmd) {
-    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir";
+    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --outdir $outdir";
     $command   .= " --dbname $dbname" if $dbname;
     $command   .= " --dbserver $dbserver" if $dbserver;

branches/czw_branch/20100427/pstamp/src/ppstampMakeStamp.c

@@ -36 +36 @@
     double d_before = check->sky->d;
     psLogMsg("ppstampMakeStamp", 2, "Before fit  Center Pixel RA: %f   DEC: %f (degrees)\n",
-             r_before * PS_DEG_RAD, d_before * PS_DEG_RAD);
+             r_before * PS_DEG_RAD, d_before * PS_DEG_RAD);
 
     if (outFPA->toSky->type != PS_PROJ_TAN) {
-        // we need to make astrometry terms for the output which are centered on the output chip center,
-        // but we keep the original plate scale
-        psFree(outFPA->toSky);
-        outFPA->toSky = psProjectionAlloc (r_before, d_before, inFPA->toSky->Xs, inFPA->toSky->Ys, PS_PROJ_TAN);
-
-        if (!pmAstromLinearizeToSky(inFPA, inChip, outFPA, outChip, roi)) {
+        // we need to make astrometry terms for the output which are centered on the output chip center,
+        // but we keep the original plate scale
+        psFree(outFPA->toSky);
+        outFPA->toSky = psProjectionAlloc (r_before, d_before, inFPA->toSky->Xs, inFPA->toSky->Ys, PS_PROJ_TAN);
+
+        if (!pmAstromLinearizeToSky(inFPA, inChip, outFPA, outChip, roi)) {
             psFree(check);
             psError(PS_ERR_UNKNOWN, false, "Failed to linearize astrometry\n");
@@ -56 +56 @@
 
         psLogMsg("ppstampMakeStamp", 2, "After fit:  Center Pixel RA: %f   DEC: %f (degrees)\n",
-                 r_after * PS_DEG_RAD, d_after * PS_DEG_RAD);
-        psLogMsg("ppstampMakeStamp", 2, "Error in fit to astrometry %.2f    %.2f (arcseconds)\n", 
-                 (r_after - r_before) *  PS_DEG_RAD * 3600, (d_after - d_before) * PS_DEG_RAD * 3600);
+                 r_after * PS_DEG_RAD, d_after * PS_DEG_RAD);
+        psLogMsg("ppstampMakeStamp", 2, "Error in fit to astrometry %.2f    %.2f (arcseconds)\n",
+                 (r_after - r_before) *  PS_DEG_RAD * 3600, (d_after - d_before) * PS_DEG_RAD * 3600);
 
         // XXX: should we fail if the fit is bad ??
@@ -238 +238 @@
         return PS_EXIT_DATA_ERROR;
     }
-    char *fpaName = psMetadataLookupStr(NULL, input->fpa->concepts, "FPA.OBS"); // Name of FPA
     pmFPAview *outview = pmFPAviewAlloc(0);
-    pmFPAAddSourceFromView(output->fpa, fpaName, outview, output->format);
+    pmFPAAddSourceFromView(output->fpa, outview, output->format);
 
     outview->chip = 0;

branches/czw_branch/20100427/pstamp/src/ppstampMosaic.c

@@ -24 +24 @@
     pmChip  *inChip = pmFPAviewThisChip(view, input->fpa);
     if (!mChip->hdu && !mChip->parent->hdu) {
-        const char *name = psMetadataLookupStr(&status, input->fpa->concepts, "FPA.OBS"); // Name of FPA
-        pmFPAAddSourceFromView(mosaic->fpa, name, mosaicView, mosaic->format);
+        pmFPAAddSourceFromView(mosaic->fpa, mosaicView, mosaic->format);
     }
     psFree(mosaicView);

branches/czw_branch/20100427/pstamp/test/detect_query_create

@@ -16 +16 @@
      $output,                   # Name of output table
      $query_id, 
+     $nostage,
      );
 
@@ -22 +23 @@
            'output|o=s'   => \$output,
            'query_id|q=s'  => \$query_id,
+           'nostage'      => \$nostage,
 ) or pod2usage( 2 );
 
@@ -66 +68 @@
 #        }
 ];
+unless(defined($nostage)) {
+    push @{ $header },        { name =>  'STAGE',
+                                writetype => TSTRING,
+                                comment => 'processing stage to examine',
+                                value => undef
+    };
+}
 
 # Specification of columns to write