Changeset 33415 for branches/meh_branches/ppstack_test/pstamp/scripts

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/detect_query_read (modified) (3 diffs)
• pstamp/scripts/detectability_respond.pl (modified) (15 diffs)
• pstamp/scripts/dquery_finish.pl (modified) (6 diffs)
• pstamp/scripts/dquery_job_run.pl (copied) (copied from trunk/pstamp/scripts/dquery_job_run.pl )
• pstamp/scripts/dqueryparse.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (10 diffs)
• pstamp/scripts/pstamp_cleanup.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (7 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (10 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_server_status (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)

branches/meh_branches/ppstack_test/pstamp/scripts/Makefile.am

@@ -29 +29 @@
         detect_query_create \
         detect_response_create \
+        dquery_job_run.pl \
         dquery_finish.pl \
         dqueryparse.pl \

branches/meh_branches/ppstack_test/pstamp/scripts/detect_query_read

@@ -222 +222 @@
     if (($colData{FPA_ID}[$i] eq "Not_Set")&&(($colData{'FILTER'}[$i] ne "Not_Set")&&
                                               ($colData{'MJD-OBS'}[$i] ne "Not_Set"))) {
-        my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
-        my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
+        my $mjd_obs = int $colData{'MJD-OBS'}[$i];
+        my $dateobs_begin = mjd_to_dateobs($mjd_obs);
+        my $dateobs_end = mjd_to_dateobs($mjd_obs + 1);
         my $ra = $colData{'RA1_DEG'}[$i];
         my $dec = $colData{'DEC1_DEG'}[$i];
@@ -229 +230 @@
         my $cmd;
         if ($colData{STAGE}[$i] eq 'SSdiff') {
-            $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
-            $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
-            $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 4";
+            # Assumption: All Stack Stack diffs are for a medium deep field
+            # go find which one the coordinates are in
+            # XXX: If we don't find one we'll just get the first SSdiff
+            # in the filter on the day in question
+            my $field = find_md_field($ra, $dec, 1.6);
+            $cmd = "$difftool -listssrun -dbname $dbname -filter $filter ";
+            $cmd .= " -mjd_obs_begin $mjd_obs -mjd_obs_end " . (1 + $mjd_obs);
+            $cmd .= " -limit 1";
+            $cmd .= " -tess_id $field%" if $field;
         }
         elsif ($colData{STAGE}[$i] eq 'WSdiff') {
@@ -366 +373 @@
 
 
+# Given an RA and  DEC return the MD field that contains it if any.
+# This is much faster than actuallly examining the tessellation
+# using dvoImagesAtCoords
+
+sub find_md_field {
+    my $ra_deg = shift;
+    my $dec_deg = shift;
+    my $radius_deg = shift;
+
+    my @MD     = qw(MD01  MD02    MD03     MD04      MD05    MD06      MD07    MD08    MD09     MD10);
+    my @MD_RA  = (35.875, 53.10, 130.592, 150.000, 161.917, 185.000, 213.454, 242.787, 333.98, 352.312);
+    my @MD_DEC = (-4.25,  -27.8,   44.317,   2.20,   58.083,  47.117,  53.243, 54.95,    0.283, -0.433);
+    my $PS_DEG_RAD = 0.017453292519943;
+
+    my $ra     = $ra_deg * $PS_DEG_RAD;
+    my $dec    = $dec_deg * $PS_DEG_RAD;
+    my $radius = $radius_deg * $PS_DEG_RAD;
+
+    for (my $i = 0; $i < scalar @MD; $i++) {
+        
+        my $dist = acos((cos($PS_DEG_RAD * $MD_DEC[$i]) * cos($dec) * cos($PS_DEG_RAD * $MD_RA[$i] - $ra)) +(sin($PS_DEG_RAD * $MD_DEC[$i]) * sin($dec)));
+
+        if ($dist < $radius) {
+            return $MD[$i];
+        }
+    }
+    print STDERR "failed to find MD field containing $ra_deg $dec_deg\n";
+
+    return undef;
+}
+
+
 __END__

branches/meh_branches/ppstack_test/pstamp/scripts/detectability_respond.pl

@@ -20 +20 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
-                       $PS_EXIT_UNKNOWN_ERROR
-                       $PS_EXIT_SYS_ERROR
-                       $PS_EXIT_CONFIG_ERROR
-                       $PS_EXIT_PROG_ERROR
-                       $PS_EXIT_DATA_ERROR
-                       $PS_EXIT_TIMEOUT_ERROR
-                       metadataLookupStr
-                       metadataLookupBool
-                       caturi
-                       );
+use PS::IPP::Config qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
+
 use Astro::FITS::CFITSIO qw( :constants );
 Astro::FITS::CFITSIO::PerlyUnpacking(1);
@@ -41 +33 @@
 my $EXTVER = 1.0;
 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
-my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
 GetOptions(
@@ -47 +39 @@
     'output=s'        =>      \$output,
     'workdir=s'       =>      \$workdir,
+    'job_id=s'        =>      \$job_id,
     'dbname=s'        =>      \$dbname,
     'dbserver=s'      =>      \$dbserver,
@@ -74 +67 @@
 
 my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
 
 if (!$dbserver) {
@@ -108 +103 @@
 } # End reading wisdom.
 else {
-#
-# Parse input request file using detect_query_read (as it's already written).
-#
+    #
+    # Parse input request file using detect_query_read (as it's already written).
+    #
     my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -123 +118 @@
     {
         my @column_names = ();
+        # split output into lines skip until the line which lists the column names is found
+        # Parse subsequent lines
         foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
             if ($entry =~ /^#/) {
@@ -128 +125 @@
                 shift(@column_names);  # Dump the hash sign.
             }
-            else {
+            elsif (scalar @column_names) {
                 # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
                 my %row_data;
                 @row_data{@column_names} = (split /\s+/, $entry);
                 for (my $i = 0; $i <= $#column_names; $i++) {
-                push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
-                $Nrows = scalar(keys(%query));
-#               print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
-            }
-            }
-        }
-    }
-#
-# Identify target images.  This should properly collate targets on a single imfile.
-#
+                    push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
+                    $Nrows = scalar(keys(%query));
+                    # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
+                }
+            }
+        }
+    }
+    #
+    # Identify target images.  This should properly collate targets on a single imfile.
+    #
     foreach my $fpa_id (keys %query) {
         my %temp_hash;
@@ -212 +209 @@
         }
         elsif ($fpa_id eq 'Not_Set') {
+            # no matching file was found skip
             next;
         }
@@ -227 +225 @@
             $need_magic = 0;
         }
-        my $mjd_min = $mjd;
+        my $mjd_min = int $mjd;
         my $mjd_max = $mjd + 1;
         
+        my $req_filter;
+        if ($filter ne 'Not_Set') {
+            $req_filter = $filter . '%';
+        }
         
         # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
@@ -237 +239 @@
                                           $fpa_id,undef,undef,
                                           $option_mask,$need_magic,
-                                          # $ra,$dec,
-                                          $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose);  
+                                          $mjd_min,$mjd_max,$req_filter,undef,$verbose);  
         
         foreach my $this_image_ref (@{ $pstamp_images_ref }) {
-            foreach my $key (sort (keys %{ $this_image_ref } )) {
-                my $value = ${ $this_image_ref }{$key};
-                if ($key eq 'row_index') {
-                    $value = join ' ', @{ $this_image_ref->{$key} };
-                }
-
-#               print "$this_image_ref $key $value\n";
-                foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
-                    $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
-                    $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
-                    $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
-                    $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
-                    $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
-                    $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
-                    $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
-                    $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
-                    
-                    if (exists($this_image_ref->{astrom})) {
-                        $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
-                    }
-                    else {
-                        $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
-                    }
-                    if (exists($this_image_ref->{class_id})) {
-                        $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
-                        $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
-                        
-                    }
-                    else {
-                        $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
-                        $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
-                    }
-                    
-                    $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
-                    if (exists($this_image_ref->{data_state})) {
-                        $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
-                    }
-                    else {
-                        $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
-                    }
-                    $query{$fpa_id}{FAULT}[$valid_index] = 0;
+
+            if (0) {
+                foreach my $key (sort (keys %{ $this_image_ref } )) {
+                    my $value = ${ $this_image_ref }{$key};
+                    if ($key eq 'row_index') {
+                        $value = join ' ', @{ $this_image_ref->{$key} };
+                    }
+                    print "$this_image_ref $key $value\n";
+                }
+            }
+
+
+            foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
+                $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
+                $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
+                $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
+                $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
+                $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
+                $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
+                $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
+                $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
+                
+                if (exists($this_image_ref->{astrom})) {
+                    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
+                }
+                else {
+                    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
+                }
+                if (exists($this_image_ref->{class_id})) {
+                    $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
+                    $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
+                    
+                }
+                else {
+                    $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
+                    $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
+                }
+                
+                $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
+                if (exists($this_image_ref->{data_state})) {
+                    $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
+                }
+                else {
+                    $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
+                }
+                $query{$fpa_id}{FAULT}[$valid_index] = 0;
                     $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
-                    if ($stage eq 'chip') {
-                        $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
-                    }
-                    
-                    # Determine if the data exists.
-                    if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 
-                        ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
-                        ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 
-                        ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 
-                        ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 
-                        ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
-                        
-                        # image is gone and it's not coming back
-                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
-                    }
-                    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
-                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
-                    }
-                    elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {                   
-                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
-                        
-                        # updating stacks isn't implemented
-                        if (($stage eq 'stack')) {
-                            $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
-                        }
-                        # updating old burntool data isn't implemented
-                        elsif ($stage eq 'chip') {
-                            if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 
-                                (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
-                                $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+                if ($stage eq 'chip') {
+                    $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
+                }
+                
+                # Determine if the data exists.
+                if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 
+                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 
+                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
+                    
+                    # image is gone and it's not coming back
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
+                }
+                elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
+                }
+                elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {               
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
+                    
+                    # updating stacks isn't implemented
+                    if (($stage eq 'stack')) {
+                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+                    }
+                    # updating old burntool data isn't implemented
+                    elsif ($stage eq 'chip') {
+                        if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 
+                            (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
+                            $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
                             }
-                        }
-                    } # End determining error faults.
-                }
-            }
-        }
+                    }
+                } # End determining error faults.
+            }
+        }
     }
 } # End calculating wisdom
@@ -324 +329 @@
 my %processing_request;
 
+if ($job_id) {
+    $wisdom_file = "$wisdom_file.$job_id";
+}
 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
 foreach my $fpa_id (keys %query) {
     for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
-        print WISDOM "$fpa_id\t";
-        foreach my $key (keys %{ $query{$fpa_id} }) {
-            print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
-        }
-        print WISDOM "\n";
-        if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
-            @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
-                ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
-                 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
-            push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
-        }
+        print WISDOM "$fpa_id\t";
+        foreach my $key (keys %{ $query{$fpa_id} }) {
+            print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
+        }
+        print WISDOM "\n";
+        my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
+        if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
+            if ($data_state ne 'full') {
+                
+                @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
+                    ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
+                     $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+            }
+            push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+        }
     }
 }
 close(WISDOM);
 
-# If there is anything that needs to be updated, create the update request list, and then exit the program.
-my $exit_code = 0;
-my $update_request_file = "${workdir}/update_request.dat";
-open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
-foreach my $images (keys %update_request) {
-    foreach my $fault (keys %{ $update_request{$images} }) {
-        if ($fault == 25) {
-            $exit_code = 25;
-        }
-        elsif ($fault != 0) {
-            $exit_code = 21;
-        }
-        my $update_request = join ' ', @{ $update_request{$images}{$fault} };
-        print UPDATE_REQUEST "$update_request\n";
-    }
-}
-close(UPDATE_REQUEST);
-if ($exit_code != 0) {
-    exit($exit_code);
+if (! defined $job_id) {
+    # If there is anything that needs to be updated, create the update request list, and then exit the program.
+    my $exit_code = 0;
+    my $update_request_file = "${workdir}/update_request.dat";
+    open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
+    foreach my $images (keys %update_request) {
+        foreach my $fault (keys %{ $update_request{$images} }) {
+            if ($fault == 25) {
+                $exit_code = 25;
+            }
+            elsif ($fault != 0) {
+                $exit_code = 21;
+            }
+            my $update_request = join ' ', @{ $update_request{$images}{$fault} };
+            print UPDATE_REQUEST "$update_request\n";
+        }
+    }
+    close(UPDATE_REQUEST);
+    if ($exit_code != 0) {
+        exit($exit_code);
+    }
 }
 
@@ -388 +402 @@
         my $weight= $query{$fpa_id}{WEIGHT}[$index];
         my $stage = $query{$fpa_id}{STAGE}[$index];
-        # if there's a fault, then we can't process this image.
-        if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) {
-            $query{$fpa_id}{PROC_ERROR}[$index] = 23;
-            
-            $query{$fpa_id}{NPIX}[$index] = 0;
-            $query{$fpa_id}{QFACTOR}[$index] = 0.0;
-            $query{$fpa_id}{FLUX}[$index] = 0.0;
-            $query{$fpa_id}{FLUX_SIG}[$index] = 0.0;
+        my $stage_id = $query{$fpa_id}{STAGE_ID}[$index];
+        my $component = $query{$fpa_id}{COMPONENT_ID}[$index];
+        # print "Input is from $stage $stage_id $component\n";
+
+        # if there's a fault or quality problem, then we can't process this image.
+        # if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) {
+        if ($fault = check_component($stage, $stage_id, $component, $imagedb)) {
+            foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+                $query{$fpa_id}{PROC_ERROR}[$i] = $fault;
+                
+                $query{$fpa_id}{NPIX}[$i] = 0;
+                $query{$fpa_id}{QFACTOR}[$i] = 0.0;
+                $query{$fpa_id}{FLUX}[$i] = 0.0;
+                $query{$fpa_id}{FLUX_SIG}[$i] = 0.0;
+            }
 
             next;
@@ -695 +716 @@
             $inHeader->{FPA_ID}->{value} = $fpa_id;
         }
+
+        my @keysa = keys %{$query{$fpa_id}};
+
+        print "$fpa_id: @keysa\n";
         
         push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
@@ -710 +735 @@
     $status = 0;
 #    print "$output\n";
+    if (-e $output) {
+        unlink $output or die "failed to unlink existing response file $output\n";
+    }
     my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
     check_fitsio( $status );
@@ -779 +807 @@
 }
 
+sub check_component {
+    my ($stage, $stage_id, $component, $imagedb) = @_;
+
+    print "Checking status of component $stage $stage_id $component\n";
+
+    my $command;
+    if ($stage eq 'diff') {
+        $command = "difftool -diffskyfile -diff_id $stage_id -skycell_id $component";
+    } elsif ($stage eq 'stack') {
+        $command = "stacktool -sumskyfile -stack_id $stage_id";
+    } elsif ($stage eq 'warp') {
+        $command = "warptool -warped -warp_id $stage_id -skycell_id $component";
+    } elsif ($stage eq 'chip') {
+        $command = "chiptool -processedimfile -chip_id $stage_id -class_id $component";
+    } else {
+        die("check_component not implemented for stage $stage yet");
+    }
+
+    $command .= " -dbname $imagedb";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my $rc = $error_code >> 8;
+        carp "$command failed $error_code $rc";
+        exit $PS_EXIT_UNKNOWN_ERROR;
+    }
+    my $output = join "", @$stdout_buf;
+    my $metadata = $mdcParser->parse($output);
+    my $results = parse_md_list($metadata);
+    if (scalar @$results != 1) {
+        carp "$command returned too many components: " . scalar @$results;
+        exit $PS_EXIT_UNKNOWN_ERROR;
+    }
+    my $it = $results->[0];
+
+    if ($stage eq 'stack') {
+        $it->{data_state} = $it->{state};
+    }
+
+    my $return_status = 0;
+    if ($it->{quality}) {
+        print "  Bad quality: $it->{quality}\n";
+        $return_status = $PSTAMP_GONE;
+    } elsif ($it->{fault}) {
+        print "  Faulted: $it->{fault}\n";
+        $return_status = $PSTAMP_GONE;
+    } elsif ($it->{data_state} ne 'full') {
+        # XXX does this work for stack?
+        carp "  Faulted: $it->{data_state}\n";
+        $return_status = $PSTAMP_GONE;
+    } else {
+        print "  Component ok.\n";
+    }
+
+    return $return_status;
+}
+
 sub my_die {
     my $message = shift;

branches/meh_branches/ppstack_test/pstamp/scripts/dquery_finish.pl

@@ -13 +13 @@
 use File::Temp qw( tempfile );
 use File::Copy;
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
 
 use PS::IPP::Metadata::Config;
@@ -66 +68 @@
 if ($product eq "NULL") {
     # parsing failed just with fault = 0 (this leaves previously set fault in place
-    stop_request($req_id, 0, $verbose);
+    update_request($req_id, 0, $verbose);
     exit 0;
 }
-if (0) {
-my $prod_dir = "$outputDataStoreRoot/$product";
-if (! -e $prod_dir ) {
-    # something must have gone wrong at the parse stage
-    stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
-    die "product directory does not exist $prod_dir";
-}
-my $outdir = "$prod_dir/$req_name";
-}
+
 if (! -e $outdir ) {
     # something must have gone wrong at the parse stage
     print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
     if (! mkdir $outdir ) {
-        stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+        update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
         die "cannot create output directory $outdir";
     }
 } elsif (! -d $outdir) {
-    stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     die "output fileset directory $outdir exists but is not a directory";
 }
@@ -97 +91 @@
 {
     my $command = "$pstamptool -listjob -req_id $req_id";
+    $command .= " -jobType detect_query";   # temporary
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -117 +112 @@
 }
 
-# XXX: have the jobs produce the reglist as with postage stamp requests
-my ($REGLIST, $reg_list) = tempfile("$outdir/reqlist.XXXX", UNLINK => !$save_temps);
-
+my ($REGLIST, $reg_list) = tempfile("$outdir/reglist.XXXX", UNLINK => !$save_temps);
+my $response_file = "response.$req_id.fits";
+my $response_path = "$outdir/$response_file";
+
+print $REGLIST "$response_file|||table|\n";
+
+my @columns = qw(ROWNUM PROC_ERROR NPIX QFACTOR FLUX FLUX_SIG FPA_ID STAGE STAGE_ID COMPONENT CMFFILE);
+my %colData;
+foreach my $col (@columns) {
+    $colData{$col} = [];
+}
+    
+my $i = 0;
 foreach my $job (@jobs) {
     my $job_id = $job->{job_id};
-    my $response_file = "response${job_id}.fits";
-    my $response_path = "$outdir/$response_file";
-
-    if (-e $response_path) {
-        # the job generated a response file put it into the Data Store
-        print $REGLIST "$response_file|||table|\n";
-
-        # do the same if we have an error file.  Should the parse data be uploaded as well?
-        my $err_file = "parse_error.txt";
-        if (-e "$outdir/$err_file") {
-            print $REGLIST "$err_file|||text|\n";
-        }
-    } else  {
-        print STDERR "detect_query response file for job $job_id not found\n" if $verbose;
-        $request_fault = $PS_EXIT_UNKNOWN_ERROR;
-    }
+
+    my $outputBase = $job->{outputBase};
+    my $results = "${outputBase}results.txt";
+
+    read_results($results, \%colData);
+
+    my $cmf = $colData{CMFFILE}->[$i];
+    print $REGLIST "$cmf|||table|\n";
+
+    $i++;
 }
 close $REGLIST;
+
+write_response_file($response_path, $req_name, \%colData);
 
 if (-s $reg_list) {
@@ -157 +158 @@
 }
 
-stop_request($req_id, $request_fault, $verbose);
+update_request($req_id, $request_fault, $verbose);
 
 exit 0;
 
-sub stop_request {
+sub read_results {
+    my $results_file = shift;
+    my $results = shift;
+
+    open IN, "<$results_file" or my_die("failed to open $results_file", $PS_EXIT_UNKNOWN_ERROR);
+
+    foreach my $line (<IN>) {
+        # skip header
+        next if $line =~ /ROWNUM/;
+        chomp $line;
+        my @words = split " ", $line;
+        my ($rownum, $proc_error, $npix, $qfactor, $flux, $flux_sig, $fpa_id, $stage, $stage_id, $component, $cmf) = @words;
+        push @{$results->{ROWNUM}}, $rownum;
+        push @{$results->{PROC_ERROR}}, $proc_error;
+        push @{$results->{NPIX}}, $npix;
+        push @{$results->{QFACTOR}}, $qfactor;
+        push @{$results->{FLUX}}, $flux;
+        push @{$results->{FLUX_SIG}}, $flux_sig;
+        push @{$results->{FPA_ID}}, $fpa_id;
+        push @{$results->{STAGE}}, $stage;
+        push @{$results->{STAGE_ID}}, $stage_id;
+        push @{$results->{COMPONENT}}, $component;
+        push @{$results->{CMFFILE}}, $cmf;
+    }
+    # XXX: make sure that the size of each of these arrays is the same
+}
+
+sub write_response_file {
+    my $output = shift;
+    my $query_id = shift;
+    my $results = shift;
+
+    my $columns;
+    my $headers;
+
+    my $EXTVER_IS_1 = 0 ; # (scalar(keys(%query)) == 1);
+#    print "EXTVER: $EXTVER_IS_1\n";
+    my ($FPA_ID,$MJD_OBS,$filter,$obscode,$status);
+    if ($EXTVER_IS_1 == 1) {
+        # Specification of columns to write
+        $columns = [
+            # matching rownum from detectability original request
+            { name => 'ROWNUM',   type => '20A', writetype => TSTRING }, 
+            # any errors that occurred during processing
+            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
+            # number of pixels used in hypothetical PSF for the query detection
+            { name => 'DETECT_N', type => 'V',   writetype => TULONG },
+            # detectibility, indicating the fraction of PSF pixels detetable by IPP
+            { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
+            # flux of the target source
+            { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+            # error in the flux of the target source
+            { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+            ];
+        
+        # Header translation table
+        $headers = {
+            'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 
+                            comment => 'MOPS Query ID for this batch query' },
+            'FPA_ID' => { name => 'FPA_ID',   type => TSTRING, 
+                          comment => 'original FPA_ID used at ingest' },
+#           'MJD-OBS' => { name => 'FPA_ID',   type => TSTRING, 
+#                         comment => 'original FPA_ID used at ingest' },
+#           'FILTER' => { name => 'FPA_ID',   type => TSTRING, 
+#                         comment => 'original FPA_ID used at ingest' },
+#           'OBSCODE' => { name => 'FPA_ID',   type => TSTRING, 
+#                         comment => 'original FPA_ID used at ingest' },
+        };
+    }
+    else {
+        # Specification of columns to write
+        $columns = [
+            # matching rownum from detectability original request
+            { name => 'ROWNUM',   type => '20A', writetype => TSTRING}, 
+            # any errors that occurred during processing
+            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
+            # number of pixels used in hypothetical PSF for the query detection
+            { name => 'DETECT_N', type => 'V',   writetype => TULONG },
+            # detectibility, indicating the fraction of PSF pixels detetable by IPP
+            { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
+            # flux of the target source
+            { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+            # error in the flux of the target source
+            { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+            # The FPA That would be in the header if it were to be there.
+            { name => 'FPA_ID',   type => '20A', writetype => TSTRING },
+            ];
+        
+        # Header translation table
+        $headers = {
+            'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 
+                            comment => 'MOPS Query ID for this batch query' },
+        };
+    }   
+
+    # Parse the list of columns
+    my @colNames;                   # Names of columns
+    my @colTypes;                   # Types of columns
+    my %colData;                    # Data for each column
+    my @colWriteType;                 # type to use to write
+    foreach my $colSpec ( @$columns) {
+        push @colNames, $colSpec->{name};
+        push @colTypes, $colSpec->{type};
+        push @colWriteType, $colSpec->{writetype};
+        $colData{$colSpec->{name}} = [];
+    }
+
+    my $inHeader = { };
+
+    # Hack to force the data to match the detect_response_create formats
+
+    $inHeader->{QUERY_ID}->{value} = $query_id;
+    if ($EXTVER_IS_1 == 1) {
+        my $fpa_id = $results->{FPA_ID}->[0];
+        $inHeader->{FPA_ID}->{value} = $fpa_id;
+    }
+    
+    # Fill the table columns with the data, making sure the flux is defined
+
+    @{$colData{'ROWNUM'}} = @{ $results->{ROWNUM} };
+    @{$colData{'ERROR_CODE'}} = @{ $results->{PROC_ERROR} };
+    @{$colData{'DETECT_N'}} = @{ $results->{NPIX} };
+    @{$colData{'DETECT_F'}} = @{ $results->{QFACTOR} };
+    @{$colData{'TARGET_FLUX'}} = @{ $results->{FLUX} };
+    @{$colData{'TARGET_FLUX_SIG'}} = @{ $results->{FLUX_SIG} };
+    @{$colData{'FPA_ID'}} = @{ $results->{FPA_ID} };
+
+    my $numRows = 0;
+    # Back to detect_response_create:
+    $status = 0;
+#    print "$output\n";
+    if (-e $output) {
+        unlink $output or die "failed to unlink existing response file $output\n";
+    }
+    my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
+    check_fitsio( $status );
+    $outFits->create_img( 16, 0, undef, $status );
+    check_fitsio( $status );
+    my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+    
+    $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status);
+    check_fitsio( $status );
+
+    # TODO: get the extension version number from somewhere common
+    $outFits->write_key( TINT, 'EXTVER', 2, 'Version of this Extension', $status );
+    check_fitsio( $status );
+
+    # Write the Extension keywords
+    foreach my $keyword ( keys %$headers ) {
+        my $value = $inHeader->{$keyword}->{value}; # Header keyword value
+        unless (defined $value) {
+            print "Can't find header keyword $keyword\n";
+            next;
+        }
+        $value =~ s/\'//g;
+        my $name    = $headers->{$keyword}->{name}; # New name
+        my $type    = $headers->{$keyword}->{type}; # Type
+        my $comment = $headers->{$keyword}->{comment}; # Comment
+        $outFits->write_key( $type, $name, $value, $comment, $status );
+        check_fitsio( $status );
+    }
+
+    for (my $i = 0; $i < scalar @colNames; $i++) {
+        my $colName = $colNames[$i];# Column name
+        my $writeType = $colWriteType[$i];
+        my $numRows = scalar(@{$colData{$colName}});
+        unless(defined($writeType)) {
+            print "write type undefined for $colName\n";
+        }
+        unless(defined($numRows)) {
+            print "num Rows undefined for $colName\n";
+        }
+        unless(defined($status)) {
+            print "status undefined for $colName\n";
+        }
+        unless(defined($colData{$colName})) {
+            print "col data undefined for $colName\n";
+        }
+        unless(defined($colName)) {
+            print "column name undefined for $i\n";
+        }
+        print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n";
+        $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
+        print STDERR "$writeType $i $numRows $colName $status\n";
+        check_fitsio( $status );
+    }
+    $outFits->close_file( $status );
+    return($status);
+}
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+    my $status = shift;         # Status of FITSIO calls
+
+    if ($status != 0) {
+        my $msg;                # Message to output
+        Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+        carp("CFITSIO error: $status => $msg");
+        die "CFITSIO error: $msg\n";
+    }
+}
+sub update_request {
     my $req_id = shift;
     my $fault = shift;
     my $verbose = shift;
     
-    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
-    $command   .= " -set_fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id";
+    if ($fault) {
+        $command .= " -set_fault $fault";
+    } else {
+        $command .= " -set_state stop";
+    }
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -176 +383 @@
     }
 }
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+
+    print STDERR $msg;
+
+    update_request($req_id, $fault);
+
+    exit $fault;
+}

branches/meh_branches/ppstack_test/pstamp/scripts/dqueryparse.pl

@@ -1 @@
-#!/usr/bin/env perl
-#
-# parse a MOPS_DETCTABILITY_QUERY table and create a results file
+#! /usr/bin/env perl
+#
+#
+# Create a response to a MOPS_DETECTABILITY_QUERY
+#
 #
 
 use strict;
 use warnings;
+
+use Sys::Hostname;
 use Carp;
+use File::Basename;
+use File::Copy;
+use IPC::Cmd 0.36 qw( can_run run );
 use Getopt::Long qw( GetOptions );
 use Pod::Usage qw( pod2usage );
-use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile tempdir);
+use POSIX;  # for strftime
 
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
-                       $PS_EXIT_UNKNOWN_ERROR
-                       $PS_EXIT_SYS_ERROR
-                       $PS_EXIT_CONFIG_ERROR
-                       $PS_EXIT_PROG_ERROR
-                       $PS_EXIT_DATA_ERROR
-                       $PS_EXIT_TIMEOUT_ERROR
-                       metadataLookupStr
-                       metadataLookupBool
-                       caturi
-                       );
-
-my ($no_update, $imagedb, $label);
-my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
-my ($job_id,$rownum); # stuff from the post-update world
-#
-# parse args
-#
-
+use PS::IPP::Config qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+# use Astro::FITS::CFITSIO qw( :constants );
+# Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+my $host = hostname();
+
+my $mode = 'queue_job';
+
+my ($req_id, $product, $need_magic, $missing_tools, $project, $label);
+my ($request_file, $outdir, $dbname, $dbserver, $verbose, $save_temps, $no_update);
 GetOptions(
-        'file=s'          =>      \$req_file,
-        'req_id=s'        =>      \$req_id,
-        'job_id=s'        =>      \$job_id,
-        'rownum=s'        =>      \$rownum,
-        'outdir=s'       =>      \$outdir,
-        'label=s'         =>      \$label,
-        'product=s'       =>      \$product,
-        'mode=s'          =>      \$mode,
-        'dbname=s'        =>      \$dbname,
-        'dbserver=s'      =>      \$dbserver,
-        'verbose'         =>      \$verbose,
-        'save-temps'      =>      \$save_temps,
-) or pod2usage(2);
-
-die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file or --job_id is required" if !$req_file;
-
-if ($mode ne "list_uri") {
-    die "req_id is required" if !$req_id;
-    die "outdir is required" if !$outdir;
-    die "product is required" if !$product;
-}
-my $missing_tools;
-my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1);
-my $detectresponse = can_run('detectability_respond.pl') or 
-    (warn "Can't find detectability_respond.pl" and $missing_tools = 1);
-my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1);
-
+    'file=s'          =>      \$request_file,
+    'req_id=s'        =>      \$req_id,
+    'outdir=s'        =>      \$outdir,
+    'label=s'         =>      \$label,
+    'product=s'       =>      \$product,
+    'mode=s'          =>      \$mode,
+    'dbname=s'        =>      \$dbname,
+    'dbserver=s'      =>      \$dbserver,
+    'verbose'         =>      \$verbose,
+    'save-temps'      =>      \$save_temps,
+    'no-update'       =>      \$no_update,
+    ) or pod2usage(2);
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --file --req_id --outdir --dbname",
+           -exitval => 3,
+    ) unless
+    defined $request_file and defined $outdir and defined $dbname and defined $req_id;
+
+my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1);
+my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
+my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
-    exit ($PS_EXIT_CONFIG_ERROR);
-}
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+unless (defined $verbose) {
+    $verbose = 0;
+}
+
+my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
 
 if (!$dbserver) {
-    my $ipprc = PS::IPP::Config->new();
-    $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
-}
-
-# just deal with these arguments once and for all
-$pstamptool .= " -dbname $dbname" if $dbname;
-$pstamptool .= " -dbserver $dbserver" if $dbserver;
-$detectresponse .= " --dbname $dbname" if $dbname;
-
-$no_update = 1 if $mode eq "list_job";
-
-# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing.
-if (!$job_id) {
-    my $argslist = "$outdir/parse.args";
-    open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
-    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n";
-    close ARGSLIST;
+    $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+
+$pstamptool .= " -dbname $dbname -dbserver $dbserver";
+
+# Project name is hardcoded to gpc1 for the moment.
+$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver);
+my $imagedb = $project->{dbname};
+if (!$imagedb) {
+    my_die("failed to find imagedb for project: $project", $PS_EXIT_CONFIG_ERROR);
 }
 
@@ -89 +85 @@
 my $fields_output; 
 {
-    my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
+    my $command = "echo $request_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -97 +93 @@
 my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
 
-my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
+my_die("$request_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
     if !(defined($extname) and defined($extver) and defined($req_name));
-my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
+my_die("$request_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
     if $extname ne "MOPS_DETECTABILITY_QUERY";
-my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR)
+my_die("$request_file is version $extver expecting 1 or 2", $PS_EXIT_PROG_ERROR)
     if ($extver ne 1) and ($extver ne 2);
 
-# Set up the workdir for this query.
-if (! -e $outdir ) {
-    mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR);
-} elsif (! -d $outdir ) {
-    my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR);
-}
-
-
-# Pass along the request file to the response generator.
-my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
-my $fault;
-my $data_to_update = '';
+if ($req_id) {
+    my $command = "$pstamptool -listreq  -name $req_name -not_req_id $req_id";
+    # set verbose to false so that error message about request not found doesn't appear in parse_error.txt
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    if ($success) {
+        # -listreq succeeded duplicate request name
+        print STDERR "QUERY_ID $req_name has already been used\n";
+        $command = "$pstamptool -addjob -job_type none -rownum 0 -state stop -fault $PSTAMP_DUP_REQUEST -req_id $req_id";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            if (!$success) {
+                my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print STDERR "skipping $command\n";
+        }
+        
+        my $datestr = strftime "%Y%m%d%H%M%S.$req_id", gmtime;
+        $req_name = "ERROR.$datestr";
+    }
+    $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name -set_outProduct $product";
+    unless ($no_update) {
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print STDERR "skipping $command\n";
+    }
+    exit (0);
+}
+
+
+
+#
+# query is a hash which uses $fpa_id as the key
+# The values are hash references which have the various parameter name as the key
+#   The values of the parameter hashes are arrays which contain the values for individual rows 
+#   in the detectabilty query request
+my %query = ();
+
 {
-    my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
-    $command .= " --save-temps" if $save_temps;
-    $command .= " --verbose" if $verbose;
-
+    #
+    # Parse input request file using detect_query_read
+    # result is a simple text file
+    #
+    my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+        run(command => $dqr_command, verbose => $verbose);
     unless ($success) {
-        warn("Warn! Unable to perform $command error code: $error_code");
-    }
-    # Use the fault code to see if we can regenerate the missing data.
-    $fault = $error_code >> 8;
-    if ($fault == $PSTAMP_NOT_AVAILABLE) {
-        unless (-e "$outdir/update_request.dat") {
-            my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
-        }
-    }   
-    
-}
-
-my $result;
-# If we returned correctly with a valid response file, get a job ID
-# for the completed work, and move the response to a standardized name.
-if ($fault == 0) {
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-    $command .= " -job_type detect_query -state stop -fault 0";
-    $command .= " -rownum 1";
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    if ($success) {
-        $job_id = join "", @$stdout_buf;
-        chomp $job_id;
-        if ($job_id && -e $response_file) {
-            rename $response_file, "$outdir/response${job_id}.fits";
-        }
-        $result = 0;
+        # This is a problem, because I'm not sure how we handle a failure to read something.
+        # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+        # we can't send that response back.
+        die("Unable to perform $dqr_command error code: $error_code");
+    }
+    my $Nrows = 0;
+    {
+        my @column_names = ();
+        # split output into lines skip until the line which lists the column names is found
+        # Parse subsequent lines
+        foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
+            if ($entry =~ /^#/) {
+                @column_names = split /\s+/, $entry;
+                shift(@column_names);  # Dump the hash sign.
+            }
+            elsif (scalar @column_names) {
+                # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
+                my %row_data;
+                @row_data{@column_names} = (split /\s+/, $entry);
+                for (my $i = 0; $i <= $#column_names; $i++) {
+                    push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
+                    $Nrows = scalar(keys(%query));
+                    # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
+                }
+            }
+        }
+    }
+    #
+    # Identify target components.  This should properly collate targets on a single imfile.
+    #
+    my $query_id;
+    foreach my $fpa_id (keys %query) {
+        my %temp_hash;
+        my $query_style = 'byexp';
+        my $stage;
+        my $filter;
+        my $mjd;
+        # Confirm that we only have one stage/filter/mjd
+        if ($fpa_id ne 'Not_Set') {   # We only need to check things that aren't the known odd case.
+            for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+                $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
+                $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
+                $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
+            }
+            if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
+                $stage = (keys(%{ $temp_hash{STAGE} }))[0];
+            }
+            else {
+                exit_with_failure(21,"Too many STAGEs specified");
+            }
+            if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
+                $filter = (keys(%{ $temp_hash{FILTER} }))[0];
+            }
+            else {
+                exit_with_failure(21,"Too many FILTERs specified");
+            }
+            if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
+                $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
+            }
+            else {
+                exit_with_failure(21,"Too many MJD-OBS specified");
+            }
+        }
+
+        # Set up a rowList with default values
+        my @rowList;
+        for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+            $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i];
+            $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i];
+            $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
+            $rowList[$i]->{COORD_MASK} = 0;
+            # Set default values
+            $query{$fpa_id}{BAD_COMPONENT}[$i] = 1;
+            $query{$fpa_id}{IMAGE}[$i] = 'no_image';
+            $query{$fpa_id}{MASK}[$i] = 'no_mask';
+            $query{$fpa_id}{WEIGHT}[$i] = 'no_weight';
+            $query{$fpa_id}{PSF}[$i] = 'no_psf';
+
+            $query{$fpa_id}{STAGE_ID}[$i] = 'no_id';
+            $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb';
+            $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic';
+            $query{$fpa_id}{MAGICKED}[$i] = 'no_magic';
+            $query{$fpa_id}{CATALOG}[$i] = 'no_catalog';
+            $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component';
+            $query{$fpa_id}{CLASS_ID}[$i] = 'no_class';
+
+            $query{$fpa_id}{STATE}[$i] = 'no_state';
+            $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate';
+            $query{$fpa_id}{FAULT}[$i] = 'no_fault';
+            $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
+        }
+
+        # Determine the query style for this fpa_id
+        if ($fpa_id =~ /o.*g.*o/) {
+            $query_style = 'byexp';
+        }
+        elsif ($fpa_id =~ /\d+/) {
+            $query_style = 'byid';
+        }
+        elsif ($fpa_id eq 'Not_Set') {
+            # no matching file was found skip
+            next;
+        }
+        else {
+            exit_with_failure(21,"Parse error in request file");
+        }
+        # Set common request components
+        my $option_mask |= 1;
+        $option_mask |= $PSTAMP_SELECT_IMAGE;
+        $option_mask |= $PSTAMP_SELECT_MASK;
+        $option_mask |= $PSTAMP_SELECT_VARIANCE;
+        $option_mask |= $PSTAMP_SELECT_PSF;
+
+        # magic is dead
+        my $need_magic = 0;
+        if ($stage eq 'stack') {
+            $need_magic = 0;
+        }
+        my $mjd_min = int $mjd;
+        my $mjd_max = $mjd + 1;
+        
+        my $req_filter;
+        if ($filter ne 'Not_Set') {
+            $req_filter = $filter . '%';
+        }
+        
+        # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
+        my $pstamp_images_ref = locate_images($ipprc,$imagedb,
+                                          \@rowList,
+                                          $query_style,$stage,
+                                          $fpa_id,undef,undef,
+                                          $option_mask,$need_magic,
+                                          $mjd_min,$mjd_max,$req_filter,undef,$verbose);  
+        
+        foreach my $this_image_ref (@{ $pstamp_images_ref }) {
+
+            # loop over the rows that this component matched
+            foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
+                $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
+                $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
+                $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
+                $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
+                $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
+                $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
+                $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
+                $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
+                
+                if (exists($this_image_ref->{astrom})) {
+                    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
+                }
+                else {
+                    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
+                }
+                if (exists($this_image_ref->{class_id})) {
+                    $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
+                    $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
+                    
+                }
+                else {
+                    $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
+                    $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
+                }
+                
+                $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
+                if (exists($this_image_ref->{data_state})) {
+                    $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
+                }
+                else {
+                    $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
+                }
+                $query{$fpa_id}{FAULT}[$valid_index] = 0;
+                    $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
+                if ($stage eq 'chip') {
+                    $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
+                }
+                
+                # Determine if the data exists.
+                if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 
+                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 
+                    ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 
+                    ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
+                    
+                    # image is gone and it's not coming back
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
+                }
+                elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
+                }
+                elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {               
+                    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
+                    
+                    # updating stacks isn't implemented
+                    if (($stage eq 'stack')) {
+                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+                    }
+                    # updating old burntool data isn't implemented
+                    elsif ($stage eq 'chip') {
+                        if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 
+                            (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
+                            $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+                            }
+                    }
+                } # End determining error faults.
+            }
+        }
+    }
+}
+
+my %update_request;
+my %rows_for_component;
+my %faulted_rows;
+
+# now build a hash using image name as key which contains the list of rows for that image
+foreach my $fpa_id (keys %query) {
+    for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
+        if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
+            push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+        } else {
+            $faulted_rows{$query{$fpa_id}{ROWNUM}[$i]} = $query{$fpa_id}{FAULT}[$i];
+        }
+    }
+}
+
+#
+# Finally build the parameter lists and queue jobs for each component
+#
+
+my $job_num = 1;
+foreach my $fpa_id (keys %rows_for_component) {
+    foreach my $component (keys % {$rows_for_component{$fpa_id} } ) {
+        my $i = $rows_for_component{$fpa_id}{$component}[0];
+        my $dep_id = 0;
+        my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
+        if ($data_state ne 'full') {
+            # Queue this data to be updated
+            my $stage = $query{$fpa_id}{STAGE}[$i];
+            my $stage_id = $query{$fpa_id}{STAGE_ID}[$i];
+            my $component = $query{$fpa_id}{COMPONENT_ID}[$i];
+            print "Need to UPDATE $stage $stage_id $component from $data_state\n";
+            $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb);
+        }
+
+        my $outputBase = "$outdir/$job_num" . "_";
+
+        # params_file is an mdc description containing the parameters for the job
+        my $params_file = $outputBase . "params.mdc";
+
+        open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR);
+
+        # first the job params
+        print PARAMS "dqueryJobParams METADATA\n";
+        foreach my $key (keys %{ $query{$fpa_id} } ) {
+            my $value = $query{$fpa_id}{$key}[0];
+            print PARAMS "\t$key\t\tSTR\t$value\n";
+        }
+        print PARAMS "END\n\n";
+
+        # then one structure with the coordinates for each row
+        print PARAMS "dqueryCoord MULTI\n";
+
+        my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG);
+        foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) {
+            print PARAMS "\ndqueryCoord METADATA\n";
+            foreach my $key (@coordkeys) {
+                my $value = $query{$fpa_id}{$key}[$row];
+                print PARAMS "\t$key\t\tSTR\t$value\n";
+            }
+            print PARAMS "END\n";
+        }
+        close(PARAMS);
+
+        # use first rownum for this component as the rownum for the job
+        my $rownum = $query{$fpa_id}{ROWNUM}[$rows_for_component{$fpa_id}{$component}[0]];
+
+        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state run -rownum $rownum";
+        $command .= " -dep_id $dep_id" if $dep_id;
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            if ($success) {
+                my $job_id = join "", @$stdout_buf;
+                print "   Queued job: $job_id\n";
+            } else {
+                my $exit_status = $error_code >> 8;
+                $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status;
+                my_die("Unable to perform $command error code: $error_code", $exit_status);
+            }
+        } else {
+            print STDERR "skipping $command\n";
+        }
+        $job_num++;
+    }
+}
+
+foreach my $rownum (keys %faulted_rows ) {
+    my $outputBase = "$outdir/$job_num" . "_";
+    my $fault = $faulted_rows{$rownum};
+    $fault = $PSTAMP_NO_IMAGE_MATCH if ($fault eq 'no_fault');
+    print STDERR "insert faulted job for $rownum: $fault\n";
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state stop -rownum $rownum -fault $fault";
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        if ($success) {
+            my $job_id = join "", @$stdout_buf;
+            print "   Queued job: $job_id\n";
+        } else {
+            my $exit_status = $error_code >> 8;
+            $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status;
+            my_die("Unable to perform $command error code: $error_code", $exit_status);
+        }
     } else {
-        warn("Unable to perform $command error code: $error_code");
-        $result = $error_code >> 8;
-    }
-}
-elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
-    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
-    # Get the dependency id for the data we're requesting be updated.
-    open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
-    while (<UPDATE_REQUEST>) {
-        my $data_to_update = $_;
-        chomp($data_to_update);
-        my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
-
-        # Link this request to a job and link that job to any dependency
-        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-        $command .= " -job_type detect_query -state run -fault 0";
-        $command .= " -rownum 1";
-        $command .= " -dep_id $dep_id" if $dep_id;
-
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-
-        if ($success) {
-            $job_id = join "", @$stdout_buf;
-            chomp $job_id;
-            if ($job_id && -e $response_file) {
-                # We shouldn't have a response file at this stage.
-                rename $response_file, "$outdir/response${job_id}.fits";
-            }
-            $result = 0;
-        } else {
-            warn("Unable to perform $command error code: $error_code");
-            $result = $error_code >> 8;
-        }
-    }
-    close(UPDATE_REQUEST);
-}
-else {
-    my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR);
-}
-
-# This does not set the request state to stop.  That will happen with the request_finish.pl script, 
-# which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
+        print STDERR "skipping $command\n";
+    }
+    $job_num++;
+}
+
 {
     my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
-    $command .= " -set_fault $result" if $result;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
+    # $command .= " -set_fault $result" if $result;
+
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
         my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
-    }
-}
- 
+        }
+    } else {
+        print STDERR "Skipping $command\n";
+    }
+}
+
 exit 0;
 
-
-# If we have to queue an update run, do so and create a new dependent
 sub queue_update_run {
-    my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_;
-
-    my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
+    my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_;
+
+#    my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
     
     if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
@@ -248 +525 @@
     return($dep_id);
 }
-        
-    
+
 
 sub my_die {
-    my $msg = shift;
-    my $fault = shift;
-
-    carp $msg;
-
-    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
-    exit $fault;
-}
+    my $message = shift;
+    my $exit_code = shift;
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp("$message");
+    exit($exit_code);
+}
+
+sub exit_with_failure {
+    my $status = shift;
+    my $message = shift;
+    my_die($message, $status);
+}
+

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_checkdependent.pl

@@ -117 +117 @@
     }
 }
+
+# magic is no longer rquired
+$need_magic = 0;
+$magic_ok =  1;
 my $status = 0;
-if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')
-        and (!$need_magic or $magic_ok or $it->{magicked} > 0)) {
+if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')) {
 
     # This Dependency is satisfied. All done. Release the pstampJobs
@@ -197 +200 @@
     }
 
+if (0) {
     if (!$job_fault and ($stage eq 'chip')) {
         # what about "error_cleaned" ?
@@ -210 +214 @@
         }
     }
+}
 
     if ($job_fault >= $PSTAMP_FIRST_ERROR_CODE) {
@@ -240 +245 @@
         my $state = $chip->{state};
         my $data_state = $chip->{data_state};
+        my $fault = $chip->{fault};
         if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
-            ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
-
-            print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n";
+            ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop') or
+            ($fault eq $PSTAMP_GONE)) {
+
+            print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n";
             my $error_code;
             if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) {
@@ -253 +260 @@
             # caller will fault the jobs
             return $error_code;
-        } elsif ($chip->{dsRun_state} eq 'failed_revert') {
-            # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE 
-            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";
-            return $PSTAMP_NOT_AVAILABLE;
         } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
 
@@ -282 +285 @@
             } elsif ($fault_count > $max_fault_count) {
                 print "$stage $stage_id has faulted $fault_count times. Giving up\n";
-                $fault = $PSTAMP_GONE;
-                faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE);
-                return $PSTAMP_GONE;
+                # XXX: stop faulting chips for now
+#                $fault = $PSTAMP_GONE;
+#                faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE);
+                $fault = $PSTAMP_NOT_AVAILABLE;
+                return $fault;
             }
             # fault the dependent
             my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
-        } elsif ($chip->{dsFile_fault} eq $PSTAMP_GONE) {
-            print STDERR "magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} is GONE";
-            return $PSTAMP_GONE;
-        } elsif ($chip->{dsFile_fault} and ($chip->{dsFile_data_state} eq 'update')) {
-            # fault the dependent
-            my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
-        }
-    }
-
-    my $status = 0;
-    if (!$queued_update) {
-        $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state);
-    }
-
-    return $status;
+        }
+    }
+
+    return 0;
 }
 
@@ -321 +315 @@
     # The update system currently requires that data that has been magicked be destreaked
     # at chip stage so it needs magic even if the customer doesn't.
+if (0) {
     if ($metadata->{magicked} != 0) {
         $need_magic = 1;
     }
-
+}
     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
@@ -369 +364 @@
         # if chip has been magicked before require it to be magicked again
         # because the warp pending query requires it.
+if (0) {
         if ($chip->{magicked} < 0) {
             print "Input has been destreaked so we must destreak before warping\n";
             $need_magic = 1 
         }
+}
 
         if ($need_magic and ($chip->{magicked} eq 0)) {
@@ -421 +418 @@
     my $metadata = shift;
     my $rlabel = shift;     # if defined a new label for the chipRun
-    my $need_magic = shift; 
+    my $need_magic = 0; 
 
     my $diff_id   = $metadata->{diff_id};
@@ -427 +424 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
-
-    # The update system currently requires that data that has been magicked be destreaked
-    # at chip stage so it needs magic even if the customer doesn't.
-    if ($metadata->{magicked} != 0) {
-        $need_magic = 1;
-    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_cleanup.pl

@@ -23 +23 @@
 
 use PS::IPP::Config qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 
 my $req_id;
@@ -95 +96 @@
 }
 
+{
+    my $command = "$pstamptool -listfile -req_id $req_id";
+    $command   .= " -dbname $dbname" if $dbname;
+    $command   .= " -dbserver $dbserver" if $dbserver;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $files = parse_md_fast($mdcParser, $output);
+        foreach my $file (@$files) {
+            $ipprc->file_delete($file->{path}); 
+        }
+    }
+}
+
 # now go find the workdir for this request 
 # XXX: we finally *have* to store this in the database

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_get_image_job.pl

@@ -25 +25 @@
 
 my $output_base;
+my $bundleroot;
 
 my $verbose;
-my $ipprc;
+my $imagedbname;
 my $dbname;
 my $dbserver;
@@ -41 +42 @@
         'rownum=s'        =>      \$rownum,
         'output_base=s'   =>      \$output_base,
+        'bundleroot=s'    =>      \$bundleroot,
+        'imagedbname=s'   =>      \$imagedbname,
         'dbname=s'        =>      \$dbname,
         'dbserver=s'      =>      \$dbserver,
@@ -52 +55 @@
 
 my_die( $err, $PS_EXIT_PROG_ERROR) if $err;
+
+my $ipprc = PS::IPP::Config->new();
 
 my $params_file = $output_base . ".mdc";
@@ -91 +96 @@
 my $missing_tools;
 my $dist_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+my $pstamptool   = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -96 +102 @@
 }
 
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+
 my $outdir = dirname($output_base);
 my $basename = basename($path_base);
-my $outroot = $output_base ."_" . $basename;
 my $results_file = $output_base . ".bundle_results";
+my $outroot;
+if ($bundleroot) {
+    my (undef, undef, undef, $mday, $month, $year) = gmtime(time());
+    $month += 1;
+    $year += 1900;
+
+    # This will generate an outroot like:
+    # neb://any/pstamp/bundles/2011/11/22/14026916_1_1_o5739g0358o.353654.wrp.215092.skycell.2083.025
+
+    $outroot = $bundleroot . sprintf("/%4d/%02d/%02d/${job_id}_", $year, $month, $mday) . basename($output_base) . "_". $basename;
+} else {
+    $outroot = $output_base ."_" . $basename;
+}
 
 {
@@ -105 +126 @@
     $command .= " --results_file $results_file";
     $command .= " --component $component --path_base $path_base --outroot $outroot";
-#    XXX: we need to do some work if we want to support muggle bundles
-#    $command .= " --no_magic if $no_magic";
-    $command .= " --magicked" if $magicked;
-    $command .= " --dbname $dbname" if $dbname;
+    # DING DONG ....
+    $command .= " --no_magic";
+    $command .= " --dbname $imagedbname" if $imagedbname;
     $command .= " --verbose" if $verbose;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -147 +167 @@
 }
 
+if ($bundleroot) {
+    {
+        # delete any existing pstampFile in case this job has faulted and
+        # been reverted
+        my $command = "$pstamptool -deletefile -job_id $job_id";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("Unable to perform $command: $error_code", $error_code >> 8);
+        }
+    }
+    my $linkname = "$outdir/$file_name";
+    if (-l $linkname or -e $linkname) {
+        unlink $linkname or my_die("Failed to unlink existing symlink: $linkname", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $bundle_name = dirname($outroot) . "/$file_name";
+    my $resolved = $ipprc->file_resolve($bundle_name);
+    if (!$resolved) {
+        my_die("failed to resolve $bundle_name", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    symlink $resolved, $linkname or my_die("failed to create symlink to $resolved in $outdir", 
+        $PS_EXIT_UNKNOWN_ERROR);
+
+    my $command = "$pstamptool -addfile -job_id $job_id -path $bundle_name";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("Unable to perform $command: $error_code", $error_code >> 8);
+    }
+}
+
+
 print REGLIST "$file_name|$bytes|$md5sum|tgz|\n";
 

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_job_run.pl

@@ -59 +59 @@
 $options = 1 if !$options;
 
+# We don't need to muggle anymore. Ignore those options
+$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -77 +79 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
-my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
+my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
 my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
 my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
@@ -87 +89 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
 
 my $jobStatus;
@@ -107 +110 @@
         $nan_masked = 0;
         $muggle = 1;
+    }
+
+    if ($stage eq "raw") {
+        $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
 
@@ -214 +221 @@
     }
 
+    # MAGIC IS DEAD
+    $nan_masked = 0;
+
     my $command = "$ppstamp $outputBase $argString $fileArgs";
+    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -239 +250 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE  => "fits", 
-                           $PSTAMP_SELECT_MASK   => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
+                           $PSTAMP_SELECT_MASK     => "mk.fits",
+                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
+                           $PSTAMP_SELECT_JPEG     => "jpg");
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG);
 
         foreach my $key (keys (%extensions)) {
@@ -272 +284 @@
 } elsif ($jobType eq "get_image") {
 
-    my $uri = "";
+    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+
+    my $params = read_params_file($outputBase);
+    my $imagedb = $params->{imagedb};
+
     my $command = "$pstamp_get_image_job --job_id $job_id --output_base $outputBase --rownum $rownum";
+    $command .= " --bundleroot $pstamp_bundle_root" if $pstamp_bundle_root;
+    $command .= " --imagedb $imagedb" if $imagedb;
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;
@@ -287 +305 @@
     }
 } elsif ($jobType eq "detect_query") {
-    # detect_query jobs are basically holders to note that we need a file updated before we can continue.
-    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
-    # run it again the same way.
-    my $argslist = "$outputBase/parse.args";
-    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
-    my $argString = <ARGSLIST>;
-    close ARGSLIST;
-    chomp $argString;
+    # my $outdir = dirname($outputBase);
+    # my $argslist = "$outdir/parse.args";
+    # open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    # my $argString = <ARGSLIST>;
+    # close ARGSLIST;
+    # chomp $argString;
 
     # XXX: should we do any other sanity checking?
-    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $command = "$dqueryparse --job_id $job_id $argString";
+    # my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    my $command = "$dquery_job_run --job_id $job_id --output_base $outputBase";
     $command .= " --dbname $dbname" if $dbname;
     $command .= " --dbserver $dbserver" if $dbserver;
     $command .= " --verbose" if $verbose;
+    $command .= " --save-temps" if $save_temps;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -311 +328 @@
     } else {
         $jobStatus = $error_code >> 8;
-        my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
-    }
+        my_die("dquery_job_run.pl failed with error code: $jobStatus", $job_id, $jobStatus);
+    }
+} elsif ($jobType eq "child") {
+    # the only thing jobs of jobType child is to finish
+    $jobStatus = 0;
 } else {
     my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
@@ -597 +617 @@
 
     $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
+    if ($exit_code > 100) {
+        carp ("invalid exit code: $exit_code changing to $PS_EXIT_UNKNOWN_ERROR");
+        $exit_code = $PS_EXIT_UNKNOWN_ERROR;
+    }
     $job_state = 'run' unless $job_state;
 

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_parser_run.pl

@@ -55 +55 @@
 }
 
+$need_magic = 0;
+
 my $missing_tools;
 
@@ -89 +91 @@
     # outdir is where all of the files generated for this request are placed
     # NOTE: this location needs to be kept in sync with the web interface ( request.php )
-    my $datestr = strftime "%Y%m%d", gmtime;
+    my $datestr = strftime "%Y/%m/%d", gmtime;
     my $datedir = "$pstamp_workdir/$datestr";
     if (! -e $datedir ) {
-        mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
-            $PS_EXIT_CONFIG_ERROR);
+        my $rc = system "mkdir -p $datedir";
+        if ($rc) {
+            my $status = $rc >> 8;
+            $status = $PS_EXIT_CONFIG_ERROR if !$status;
+            my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+                $status);
+        }
     }
 
@@ -287 +294 @@
     carp($msg);
 
+    if (!$req_id) {
+        exit $PS_EXIT_CONFIG_ERROR;
+    }
+
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;

branches/meh_branches/ppstack_test/pstamp/scripts/pstamp_server_status

@@ -71 +71 @@
         $error_code = (($error_code >> 8) or 1);
         if ($error_code == 12) {
-            #print "Postage Stamp Server is not running\n";
-            print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
+            print "Postage Stamp Server is not running\n";
+            #print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
             exit 0;
         } else {

branches/meh_branches/ppstack_test/pstamp/scripts/pstampparse.pl

@@ -148 +148 @@
     } elsif ($lcname =~ /qub/) {
         $label = "QUB";
+        $label_changed = 1;
+    } elsif ($lcname =~ /pypstamp/) {
+        $label = "PY";
         $label_changed = 1;
     }
@@ -251 +254 @@
     # If we encounter an error for a particular row add a job with the proper fault code.
 
+    my $stage = $row->{IMG_TYPE};
+
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
@@ -272 +277 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp') {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' for JOB_TYPE 'get_image'\n";
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -334 +339 @@
     }
 
-    my $stage = $row->{IMG_TYPE};
     if (!check_image_type($stage)) {
         print STDERR "invalid IMG_TYPE for row $rownum\n";
@@ -460 +464 @@
     my $numRows = scalar @$rowList;
 
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $id = "" if !$id;
+    }
     print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";