Changeset 36680 for branches/eam_branches/ps2-tc3-20130727/pstamp/scripts

Timestamp:

Apr 21, 2014, 5:42:34 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/Makefile.am (modified) (2 diffs)
• pstamp/scripts/dqueryparse.pl (modified) (1 diff)
• pstamp/scripts/ftpsrequest (copied) (copied from trunk/pstamp/scripts/ftpsrequest )
• pstamp/scripts/psgetcalibinfo (copied) (copied from trunk/pstamp/scripts/psgetcalibinfo )
• pstamp/scripts/psmkreq (modified) (5 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (9 diffs)
• pstamp/scripts/pstamp_cleanup.pl (modified) (1 diff)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_insert_request.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (11 diffs)
• pstamp/scripts/pstamp_queue_cleanup.pl (copied) (copied from trunk/pstamp/scripts/pstamp_queue_cleanup.pl )
• pstamp/scripts/pstamp_save_server_status.pl (modified) (1 diff, 1 prop)
• pstamp/scripts/pstamp_server_status (modified) (2 diffs)
• pstamp/scripts/pstampparse.pl (modified) (20 diffs)

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/Makefile.am

@@ -11 +11 @@
         pstampparse.pl \
         pstamp_parser_run.pl \
+        pstamp_queue_cleanup.pl \
         pstamp_queue_requests.pl \
         pstamp_request_file \
@@ -21 +22 @@
         pstamp_get_image_job.pl \
         psmkreq \
+        psgetcalibinfo \
         psstatus \
         pstampstopfaulted \

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/dqueryparse.pl

@@ -502 +502 @@
     $command .= " -need_magic" if $need_magic;
 
-    my $rlabel = "dq_ud_" . $label if $label;
+    my $rlabel = "ps_ud_" . $label if $label;
     $command .= " -rlabel $rlabel" if $rlabel;
 

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq

@@ -39 +39 @@
 my $job_type     = 'stamp';
 my $req_type     = 'bycoord';
-my $stage        = 'chip';
+my $stage        = 'stack';
 my $option_mask;
 my $width        = $default_size;
 my $height       = $default_size;
+my $whole_file   = 0;
 my $project      = 'gpc1';
 my $coord_mask;
@@ -81 +82 @@
     'width=i'           => \$width,
     'height=i'          => \$height,
+    'whole-file'        => \$whole_file,
     'pixcenter'         => \$pixcenter,
     'arcseconds'        => \$arcseconds,
@@ -141 +143 @@
         pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 )
             if defined $x or defined $y;
-    } elsif (!$pixcenter) {
-        pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 )
-            unless (defined $ra and defined $dec);
-        # to simplify code we just use $x and $y from here
-        $x = $ra;
-        $y = $dec;
     } else {
-        pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 )
-                unless (defined $x and defined $y);
+        if ($whole_file) {
+            $pixcenter = 1;
+            $x = 0;
+            $y = 0;
+            $width = 0;
+            $height = 0;
+        }
+        if (!$pixcenter) {
+            pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 )
+                unless (defined $ra and defined $dec);
+            # to simplify code we just use $x and $y from here
+            $x = $ra;
+            $y = $dec;
+        } else {
+            pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 )
+                    unless (defined $x and defined $y);
+        }
     }
 } else {
@@ -177 +188 @@
 $id = 0 if !$id;
 
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack'
+    or $stage eq 'stack_summary') {
     die "$stage is not a valid value for stage\n";
 }
@@ -208 +220 @@
         $option_mask |= $PSTAMP_SELECT_PSF      if $psf;
         $option_mask |= $PSTAMP_SELECT_BACKMDL  if $backmdl;
+        $option_mask |= $PSTAMP_SELECT_UNCOMPRESSED if $uncompressed;
         $option_mask |= $PSTAMP_SELECT_UNCONV   if $unconvolved;
         $option_mask |= $PSTAMP_USE_IMFILE_ID   if $use_imfile_id;

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl

@@ -27 +27 @@
 my $IPP_DIFF_MODE_STACK_STACK = 4;
 
-my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
+my ($dep_id, $stage, $stage_id, $component, $imagedb, $label, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
 my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
 
@@ -36 +36 @@
     'component=s'   =>  \$component,
     'imagedb=s'     =>  \$imagedb,      # dbname for images lookups.
-    'rlabel=s'      =>  \$rlabel,
+    'label=s'       =>  \$label,        # request's label
+    'rlabel=s'      =>  \$rlabel,       # pstampDependent.rlabel (deprecated)
     'need_magic'    =>  \$need_magic,
     'fault_count=i' =>  \$fault_count,
@@ -73 +74 @@
 }
 
+if ($label) {
+    # rlabel is deprecated. Use one based on the supplied label parameter which is the current label 
+    # for the request, which may be different than the one given to the dependent when the job was parsed.
+    # XXX: having the convention that update label is 'ps_ud_' . $label of request embedded here 
+    # (and formerly in pstampparse.pl) is not particularly clean but it's simple.
+    my $new_rlabel = 'ps_ud_' . $label;
+    if ($new_rlabel ne $rlabel) {
+        print "Notice: using $new_rlabel instead of $rlabel for update label.\n";
+        $rlabel = $new_rlabel;
+    }
+}
+
+
 if (!$ps_dbserver) {
     $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
@@ -304 +318 @@
             }
             $queued_update = 1;
+        } elsif ($chip->{state} eq 'cleaned' and $chip->{data_state} eq 'update') {
+            # we've had a number of runs in this limbo state
+
+            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+            $queued_update = 1;
         } elsif ($chip->{fault}) {
             $fault_count++;
@@ -438 +468 @@
             $chips_ready = 0;
             $chip->{fault} = $chip->{chip_fault};
+            $chip->{data_group} = $chip->{chip_data_group};
             push @chipsToUpdate, $chip;
         } else {
@@ -459 +490 @@
             print "skipping $command\n";
         }
+    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
+        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+
     } elsif (scalar @chipsToUpdate > 0) {
         my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic);
@@ -528 +573 @@
             }
             return $warp_status;
+        } elsif ($warp->{quality} != 0) {
+            print STDERR "input warp has quality error\n";
+            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
+            return $PSTAMP_GONE;
         }
         # warps are ready fall through and queue the diff update
@@ -575 +624 @@
     } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
         # check the state of the input stack
-        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
+        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
         my $stack1 = runToolAndParseExpectOne($command, $verbose);
-        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
+        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
 
         if ($stack1->{state} ne 'full') {
-            print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+            print STDERR "input stack $skycell->{stack1} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
             faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
             return $PSTAMP_GONE;
@@ -590 +639 @@
 
         if ($stack2->{state} ne 'full') {
-            print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+            print STDERR "template stack $skycell->{stack2} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
             faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
             return $PSTAMP_GONE;

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_cleanup.pl

@@ -80 +80 @@
 }
 
+if (0) {
 my_die("Cleanup not yet supported for reqType: $reqType", $req_id, $PS_EXIT_UNKNOWN_ERROR)
     if ($reqType ne "pstamp") and ($reqType ne "NULL") and ($reqType ne "dquery");
+}
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_get_image_job.pl

@@ -65 +65 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $data = $mdcParser->parse(join "", (<INPUT>)) or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+my $data = $mdcParser->parse(join "", (<INPUT>)) 
+    or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+
 my $components = parse_md_list($data);
 my $n = scalar @$components;
@@ -83 +85 @@
     print STDERR "stage_id is $stage_id\n";
     print STDERR "path_base is $path_base\n";
-    print STDERR "path_base is $path_base\n";
     print STDERR "CAMERA is $camera\n";
     print STDERR "magicked is " . (defined $magicked ? $magicked : "undefined") . "\n";
@@ -89 +90 @@
 
 if (!$camera or !$path_base or !$component or !$stage_id or !$stage) {
-       my_die("failed to parse params from: $params_file", $PS_EXIT_UNKNOWN_ERROR);
+       my_die("One or more parameters are missing in: $params_file", $PS_EXIT_UNKNOWN_ERROR);
 }
 

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl

@@ -45 +45 @@
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
+my $fhead = can_run('fhead')  or (warn "Can't find fhead"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -56 +57 @@
     # Note that if it's a pstamp request then REQ_NAME should be defined.
     # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
-    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    # my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    my $command = "$fhead -x 0 $tmp_req_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-if (0) {
-    # stoopid fields doesn't set exit status to zero when it works
     unless ($success) {
         print STDERR @$stderr_buf;
         exit $error_code >> 8;
     }
-}
     my $output = join "", @$stdout_buf;
-    (undef, $extname, $extver, $req_name) = split " ", $output;
+
+
+    my $makehash = 0;
+    my %hash;
+    foreach my $line (split "\n", $output) {
+        chomp $line;
+        # split lines inte left and right using equals sign. left is the keyword
+        my ($key, $right) = split "=", $line;
+            # skip if there was no '='
+        next if !$right;
+
+        $key =~ s/ //g;
+
+        # separate value from comment
+        my ($value, $comment) = split "/", $right;
+        # remove ' and space characters from key and value
+        $value =~ s/'//g;
+        $value =~ s/ //g;
+
+        #    print "$key $value\n";
+
+        # extract the values that we are looking for
+        $extname  = $value if ($key eq "EXTNAME");
+        $extver   = $value if ($key eq "EXTVER");
+        $req_name = $value if ($key eq "REQ_NAME");
+        $req_name = $value if ($key eq "QUERY_ID");
+
+        # optionally build hash. Duplicate keys get the last value seen
+        if ($makehash) {
+            $hash{$key} = $value;
+        }
+    }
+
+    # (undef, $extname, $extver, $req_name) = split " ", $output;
+
     if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
         print STDERR "invalid request file\n";
+        print "invalid request file\n";
         exit $PS_EXIT_DATA_ERROR;
     }
+
     if (!defined $req_name) {
         print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
+        print "invalid request file no REQ_NAME or QUERY_ID\n";
+        exit $PS_EXIT_DATA_ERROR;
+    }
+    if (!defined $extver) {
+        print STDERR "invalid request file no EXTVER found\n";
+        print "invalid request file no EXTVER found\n";
         exit $PS_EXIT_DATA_ERROR;
     }
@@ -119 +160 @@
 
 exit 0;
+
 # Ask the database for the next web request number
 sub get_webreq_num

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl

@@ -57 +57 @@
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
 
-$options = 1 if !$options;
-
-# We don't need to muggle anymore. Ignore those options
-$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
+# ppstamp requires an input file
+$options = $PSTAMP_SELECT_IMAGE if !$options;
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -78 +76 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
 my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
-my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
-my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
 
@@ -96 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
     my $stage = $params->{stage};
-
-    # XXX: should we do any other sanity checking?
-    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $nan_masked = 1;
-    my $muggle = 0;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
-        # Attempt to find or create a muggle image
-        $nan_masked = 0;
-        $muggle = 1;
-    }
+    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
+
+    if ($stage eq 'stack_summary') {
+
+        # remove options not supported by stack summary
+        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE 
+            | $PSTAMP_RESTORE_BACKGROUND);
+
+        # stackSummary outputs do not follow the usual IPP conventions for file names.
+        # The parser (actually Job.pm) has deferred handling this until here
+        update_stack_summary_filenames($params);
+
+    } elsif ($stage ne 'stack') {
+        # ignore options only supported by stack and stack_summary
+        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+    }
+   
 
     if ($stage eq "raw") {
@@ -122 +126 @@
     my @file_list = ($params->{image});
     
-    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
+    if ($options & $PSTAMP_SELECT_MASK) {
         $mask = $params->{mask};
         $fileArgs .= " -mask $mask";
@@ -162 +166 @@
     # find our output directory
     my $outdir = dirname($outputBase);
-    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-    
-    if ($muggle) {
-        # first see if the original uncensored images are around
-        if (check_for_backups($params, \$fileArgs)) {
-            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
-            print "Making stamps from backup images\n";
-        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
-            # user required uncensored but since stage isn't chip we can't rebuild them
-            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
-            # we can only restore pixels for chip stage images if the data has been updated.
-            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
-            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
-            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
-            # make stamps from uncensored images
-            $muggle = 0;
+
+    my ($calib_fd, $calibfile);
+    if ($stage eq 'chip' or $stage eq 'warp') {
+        my $cam_id = $params->{cam_id};
+        if (!$cam_id) {
+            carp "no cam_id found in job params\n";
+            exit $PS_EXIT_PROG_ERROR;
+        }
+        ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps);
+        close $calib_fd;
+
+        my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        my $exitStatus;
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
         } else {
-            # Try and replace the streaks from the recovery images
-
-            my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
-            $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
-            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
-            # find the "directory" of the input path_base
-            my $inputdir = dirname($image);
-            my $base = basename($image);
-            $tmpImage = "$tmproot/$base";
-
-            @file_list = ();
-
-            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
-            # stage work. Import the rule here.
-            my $recImage = "$inputdir/REC_$base";
-            my $newFileArgs = " -file $tmpImage";
-            $muggle_command .= " -image $image -recimage $recImage";
-            push @file_list, $recImage;
-
-            if ($mask) {
-                $base = basename($mask);
-                $tmpMask = "$tmproot/$base";
-                $newFileArgs .= " -mask $tmpMask";
-                my $recMask = "$inputdir/REC_$base";
-                $muggle_command .= " -mask $mask -recmask $recMask";
-                push @file_list, $recMask;
-            }
-
-            if ($variance) {
-                $base = basename($variance);
-                $tmpVariance = "$tmproot/$base";
-                $newFileArgs .= " -variance $tmpVariance";
-                my $recVariance = "$inputdir/REC_$base";
-                $muggle_command .= " -weight $variance -recweight $recVariance";
-                push @file_list, $recVariance;
-            }
-            if (check_files(0, @file_list)) {
-                # recovery files exist and are accessible restore the excised pixels
-
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $muggle_command, verbose => $verbose);
-                unless ($success) {
-                    my $exitStatus = WEXITSTATUS($error_code);
-                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
-                }
-
-                # set the ppstamp file arguments
-                $fileArgs = $newFileArgs;
-            } else {
-                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
-                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
-                }
-                # just make stamps from the censored images
-                print "Unable to restore uncensored images, will extract stamps from censored images\n";
-                # these files won't be used so zap them
-                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
-            }
-        }
-    }
-
-    # MAGIC IS DEAD
-    $nan_masked = 0;
-
-    my $command = "$ppstamp $outputBase $argString $fileArgs";
-    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
-    $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
-    $command .= " -stage $stage";
-    $command .= " -censor_masked" if $nan_masked;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+            print STDERR "psgetcalibinfo failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        exit $exitStatus if $exitStatus;
+
+        if (-s $calibfile == 0) {
+            print "no calibration information found for $cam_id\n";
+            $calibfile = undef;
+        }
+    }
+
+    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
+    # otherwise we make copies of the input files to the outputs
+    my $use_ppstamp = ($stage ne 'stack_summary');
 
     my $exitStatus;
-    if (WIFEXITED($error_code)) {
-        $exitStatus = WEXITSTATUS($error_code);
-    } else {
-        print STDERR "ppstamp failed error_code: $error_code\n";
-        $exitStatus = $PS_EXIT_SYS_ERROR;
+    if ($use_ppstamp) {
+        my $command = "$ppstamp $outputBase $argString $fileArgs";
+        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
+        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
+        $command .= " -stage $stage";
+        $command .= " -forheader $calibfile" if $calibfile;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
+        } else {
+            print STDERR "ppstamp failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        # XXX: if stage is stack deal with EXP and NUM images if selected
+    } else {
+        $exitStatus = justCopyFiles($outputBase, \$options, $params);
     }
 
@@ -270 +230 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
-                           $PSTAMP_SELECT_MASK     => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
-                           $PSTAMP_SELECT_SOURCES  => "cmf",
-                           $PSTAMP_SELECT_JPEG     => "jpg");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits', 
+                           $PSTAMP_SELECT_MASK     => 'mk.fits',
+                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
+                           $PSTAMP_SELECT_SOURCES  => 'cmf',
+                           $PSTAMP_SELECT_JPEG     => 'jpg',
+                           $PSTAMP_SELECT_EXP      => 'exp.fits',
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -298 +266 @@
         close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP || $exitStatus == $PSTAMP_NO_VALID_PIXELS) {
+        $jobStatus = $exitStatus;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -419 +387 @@
         my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $filter = $params->{filter};
+        $filter = ' ' if !$filter;
+        $filter = substr $filter, 0, 1;
         my $cmf_file;
 #        if ($stage ne 'chip') {
@@ -431 +402 @@
             if (!$therest or !$rownum or !$jobnum);
 
-        my $prefix = "${rownum}_${jobnum}_";
+        # XXX: Here we are assuming the form of the output file names
+        # if we change this in pstampparse we'll need to remember ....
+        # (the last time I forgot)
+        my $prefix = "${rownum}_${jobnum}_${filter}_";
 
         if ($cmf_file) {
@@ -523 +497 @@
 }
 
-sub check_for_backups {
-    my $params = shift;
-    my $r_fileArgs = shift;
-
-    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
-    my $results = runToolAndParse($command, $verbose);
-    my $dsComponent  = $results->[0];
-
-    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
-
-        print "magicDSFile state is full. Backup images should exist\n";
-
-        # replace the file names with the backup paths
-        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
-        my @args = split " ", $$r_fileArgs;
-
-        my $newFileArgs;
-        while (@args) {
-            my $a = shift @args;
-            my $f = shift @args;
-            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
-
-            my $backup_image_name = $ipprc->destreaked_filename($f);
-            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
-            $newFileArgs .= " $a $backup_image_name";
-        }
-        $$r_fileArgs = $newFileArgs;
-
-        return 1;
-    }
-
-    return 0;
-}
-
 my $neb;
 sub storage_object_exists
@@ -630 +570 @@
 }
 
+# stack_summary stage does not currently use proper file rules.
+# The parser defers handlint this to us..
+
+sub update_stack_summary_filenames {
+    my $params  = shift;
+
+    my $path_base = $params->{path_base};
+
+    $params->{image}  = $path_base . ".image.b1.fits";
+    $params->{mask}   = $path_base . ".mask.b1.fits";
+    $params->{weight} = $path_base . ".variance.b1.fits";
+    $params->{jpeg}   = $path_base . ".image.0.b1.jpeg";
+    $params->{exp}    = $path_base . ".exp.b1.fits";
+    $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
+}
+
+sub myCopy {
+    my ($dest, $src, $type, $dieOnFail) = @_;
+
+    my $result;
+    my $resolved = $ipprc->file_resolve($src);
+    if ($resolved and $ipprc->file_exists($resolved)) {
+        print "Copying $src to $dest\n" if $verbose;;
+        $result = copy($resolved, $dest);
+    } else {
+        my $msg = "Specified source $type image $src not found.";
+        if ($dieOnFail) {
+            $msg .= "\n";
+        } else {
+            $msg .= " ignoring\n";
+        }
+        carp $msg;
+        $result = 0;
+    }
+    
+    if (!$result and $dieOnFail) {
+        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
+    }
+    return $result;
+}
+
+sub justCopyFiles {
+    my ($outputBase, $r_options, $params) = @_;
+
+    print "Just copying files for $job_id.\n";
+
+    my $options = $$r_options;
+    if ($options & $PSTAMP_SELECT_IMAGE) {
+        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
+    }
+    if ($options & $PSTAMP_SELECT_MASK) {
+        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
+            $options &= ~$PSTAMP_SELECT_MASK;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
+            $options =  ~$PSTAMP_SELECT_VARIANCE;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_JPEG) {
+        if (!myCopy("$outputBase.jpg", $params->{jpeg}, 'jpeg', 0)) {
+            $options &= ~$PSTAMP_SELECT_JPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXP) {
+        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXP;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUM) {
+        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
+        }
+    }
+
+    $$r_options = $options;
+
+    print "Done with copy.\n";
+
+    return 0;
+}
+
 sub my_die
 {

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_save_server_status.pl

@@ -61 +61 @@
     $year, $month, $mday, $hour, $min, $sec;
 
-my $command = "pstamp_server_status > $file";
+my $command = "pstamp_server_status --workdir $pstamp_workdir > $file";
+#my $command = "pstamp_server_status > $file";
 my  ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run(command => $command, verbose => $verbose);

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_server_status

@@ -15 +15 @@
 
 
-my ($rundir, $verbose, $save_temps);
+my ($rundir, $workdir, $verbose, $save_temps);
 my $br = '<br />';
 
 GetOptions(
     'rundir=s'      => \$rundir,
+    'workdir=s'     => \$workdir,
     'verbose'       => \$verbose,
     'save-temps'    => \$save_temps,
@@ -155 +156 @@
     }
 }
+if ($workdir) {
+    # get the space available in the working directory file system
+    my $df_output = `df -h $workdir`;
+    my @lines = split "\n", $df_output;
+    foreach my $line (@lines) {
+        next unless $line =~ /data/;
+        my ($total, $used, $free, $percent, $part) = split " ", $line;
+        print "<br><b>Server Working Directory:</b>&nbsp;&nbsp;$percent full. $free free out of $total total.<br>\n";
+    }
+
+}
 
 # now run the script psstatus to check the status of running requests and finished requests

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+my $dump_params = 0;
 
 # set this to true to disable update processing
@@ -48 +49 @@
     'save-temps'=>  \$save_temps,
     'no-update' =>  \$no_update,
+    'dump-params' => \$dump_params,
 );
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76 +80 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116 +120 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175 +184 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212 +226 @@
 }
 
-my $nRows = scalar @$rows;
+my $nRows = $rows ? scalar @$rows : 0;
 print "\n$nRows rows read from request file\n";
 
+unless ($nRows) {
+    # pstamp_job_run was invoked so the request file must have contained a valid header
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
+    print STDERR "Invalid request file.\n";
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    exit 0;
+}
+
+
+# if label is for one of the high priority channels, watch for big requests
+my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
+
+# XXX: these should be in a configuration file somewhere not hard coded
+my $big_limit = 100;
+my $job_big_limit = 400;
+
+if ($watch_for_big_requests and $nRows > $big_limit) {
+    $label = change_to_lower_priority_label($label);
+    $watch_for_big_requests = 0;
+}
 
 my $num_jobs = 0;
-my $imageList;
-my $stage;
+
 foreach my $row (@$rows) {
-
-    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
-        # this is a big request and it came from the upload page and doesn't have a specific label assigned
-        # change it to the generic one that has lower with lower priority
-        $label = 'WEB.BIG';
-        print "\nChanging label for big WEB.UP request to $label\n";
-
-        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
 
     # validate the paramaters
@@ -246 +267 @@
 
     $num_jobs += processRow($action, $row);
+
+    # see whether number of jobs limit for high priority request was exceeded
+    if ($watch_for_big_requests and $num_jobs > $job_big_limit) {
+        $label = change_to_lower_priority_label($label);
+        $watch_for_big_requests = 0;
+    }
 }
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
 exit 0;
+
+# end of main function
+
+sub change_to_lower_priority_label {
+    my $label = shift;
+    my $old_label = $label;
+    $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+    print "\nChanging label for big $old_label request to $label\n";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    return $label;
+}
 
 sub checkRow {
@@ -267 +313 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
+        $rownum = 'NULL' if !defined $rownum;
         print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -272 +319 @@
     }
     my $job_type = $row->{JOB_TYPE};
-    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+    if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) {
+        $job_type = 'NULL' if !defined $job_type;
         print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -281 +329 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        $req_type = 'NULL' if !defined $req_type;
         print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -286 +335 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
+        # get_image jobs are quite expensive in terms of space so we are currently restricting them
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' 
+            or ($req_type eq 'byskycell' and $stage eq 'stack')
+            or ($stage eq 'stack_summary')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -295 +347 @@
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        if ($job_type eq 'get_image') {
+        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
             $row->{COMPONENT} = 'all';
         } else {
@@ -384 +436 @@
     }
 
-    if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
+    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
+        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -457 +509 @@
     my $start_locate = gettimeofday();
 
-    print "\nCalling new_locate_images for row: $rownum\n";
-
-    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+    print "\nCalling locate_images_for_row for row: $rownum\n";
+
+    my $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -521 +573 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -532 +584 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -581 +603 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
+    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -701 +723 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);
 
@@ -758 +794 @@
             print "$image->{image}\n";
             ++$firstRow->{job_num};
+            if ($dump_params) {
+                my $rownum = $firstRow->{ROWNUM};
+                my $jobnum = $firstRow->{job_num};
+                my $filter = substr $image->{filter}, 0, 1;
+                my $output_base = "${rownum}_${jobnum}_${filter}";
+                write_params($output_base, $image);
+            }
         }
     } elsif ($job_type eq "get_image") {
@@ -1163 +1206 @@
     }
     if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
-        ($img_type eq "stack") or ($img_type eq "diff")) {
+        ($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
         return 1;
     } else {