Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/Makefile.am
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/Makefile.am	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/Makefile.am	(revision 36680)
@@ -11,4 +11,5 @@
 	pstampparse.pl \
 	pstamp_parser_run.pl \
+	pstamp_queue_cleanup.pl \
 	pstamp_queue_requests.pl \
 	pstamp_request_file \
@@ -21,4 +22,5 @@
         pstamp_get_image_job.pl \
 	psmkreq \
+	psgetcalibinfo \
 	psstatus \
 	pstampstopfaulted \
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/dqueryparse.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/dqueryparse.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/dqueryparse.pl	(revision 36680)
@@ -502,5 +502,5 @@
     $command .= " -need_magic" if $need_magic;
 
-    my $rlabel = "dq_ud_" . $label if $label;
+    my $rlabel = "ps_ud_" . $label if $label;
     $command .= " -rlabel $rlabel" if $rlabel;
 
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/ftpsrequest
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/ftpsrequest	(revision 36680)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/ftpsrequest	(revision 36680)
@@ -0,0 +1,203 @@
+#!/usr/bin/perl
+
+# ftpsrequest
+
+# Creates a PS1 postage stamp request table using the FTOOLS program ftcreate.
+# The input file format is one line per row with columns described in the array columnsAndTypes given
+# below. 
+#
+# Note: The input format is the same as that used for the rows by the IPP program pstamp_request_file with
+# the exception that the comment is contained in the last column as a quoted string rather than by the |
+# character.
+
+# the option --print-sample may be used to print out an example of a line in an input file to stdout.
+#
+
+# The program ftcreate is part of the FTOOLS package which is described at http://heasarc.gsfc.nasa.gov/ftools
+# and in
+# Blackburn, J. K. 1995, in ASP Conf. Ser., Vol. 77, Astronomical Data Analysis Software and Systems IV, 
+# ed. R. A. Shaw, H. E. Payne, and J. J. E. Hayes (San Francisco: ASP), 367. 
+
+use strict;
+use warnings;
+
+# using a minimal number of perl modules for ease of portablity
+
+use File::Temp qw(tempfile);
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Array describing the columns in the request table and the order of the columns in the input file
+# third column is value for sample request. A stamp for an i band stack of the tadpole galaxy.
+my @columnsAndTypes = qw(
+        ROWNUM      J   1
+        CENTER_X    D   241.516417
+        CENTER_Y    D   55.425369
+        WIDTH       D   1000
+        HEIGHT      D   1000
+        COORD_MASK  J   2
+        JOB_TYPE    16A stamp
+        OPTION_MASK J   65
+        PROJECT     16A gpc1
+        SURVEY_NAME 16A 3PI
+        IPP_RELEASE 64A null
+        REQ_TYPE    16A bycoord
+        IMG_TYPE    16A stack
+        ID          16A null
+        TESS_ID     64A null
+        COMPONENT   64A null
+        DATA_GROUP  64A null
+        REQFILT     16A i
+        MJD_MIN     D   0
+        MJD_MAX     D   0
+        RUN_TYPE    16A null
+        FWHM_MIN    D   0
+        FWHM_MAX    D   0
+        COMMENT     64A tadpole_galaxy
+);
+
+
+my ( $input,			# Name of input text file
+     $output,			# Name of output table
+     $req_name, 
+     $print_sample,
+     $email, 
+     $clobber,
+     $help,
+     $verbose,
+     $save_temps,
+     );
+
+
+GetOptions(
+	   'input|i=s'      => \$input,
+	   'output|o=s'     => \$output,
+	   'req_name|r=s'   => \$req_name,
+	   'email=s'        => \$email,
+           'help'           => \$help,
+           'print-sample'   => \$print_sample,
+           'clobber'        => \$clobber,
+	   'save-temps'     => \$save_temps,
+	   'verbose'        => \$verbose,
+) or pod2usage( 2 );
+
+my $usageMessage = "$0: --input <input> --output <output> -req_name <request_name>";
+if ($help) {
+    # pod2usage( -msg => $usageMessage, -exitval => 0);
+    pod2usage( -exitval => 0);
+}
+if ($print_sample) {
+    printSample();
+}
+
+pod2usage ( -msg => "Required options: --input --output --req_name\n", -exitval => 1)
+    unless 
+        defined $input and
+        defined $output and
+        defined $req_name;
+
+
+system "which ftcreate >& /dev/null";
+if ($?){
+    die "Can't find required program ftcreate\n";
+}
+
+# temp files for header keyword file and column descriptor file
+my ($hfile, $hfilename) = tempfile( "/tmp/hfile.XXXX", UNLINK => !$save_temps, SUFFIX => '.txt');
+my ($cfile, $cfilename) = tempfile( "/tmp/cfile.XXXX", UNLINK => !$save_temps, SUFFIX => '.txt');
+
+# create the temp files
+makeHeaderFile($hfile, $req_name, $email);
+makeColumnDescriptorFile($cfile);
+
+# here is the command that builds the fits table
+my $cmd = "ftcreate extname=PS1_PS_REQUEST headfile=$hfilename $cfilename $input $output";
+$cmd .= ' clobber=yes' if $clobber;
+
+print STDERR "$cmd\n" if $verbose;
+
+my $rc = system $cmd;
+if ($rc) {
+    my $status = $rc >> 8;
+    die "ftcreate exited with $status ($rc)\n";
+}
+
+exit 0; 
+
+############################
+
+sub makeHeaderFile {
+    my ($hfile, $req_name, $email) = @_;
+
+    $email = 'null' if !defined $email;
+
+    print $hfile "EXTVER=2 / PSTAMP request format version\n";
+    print $hfile "ACTION=PROCESS\n";
+    print $hfile "REQ_NAME=$req_name / unique name for the postage stamp request\n";
+    print $hfile "EMAIL=$email\n";
+    close $hfile;
+}
+
+
+
+sub makeColumnDescriptorFile {
+    my $cfile = shift;
+
+    my $n = scalar @columnsAndTypes;
+
+    die "ERROR: number of columnsAndTypes must be multiple of 3\n" if $n % 3;
+
+    for (my $i = 0; $i < $n; $i += 3) {
+        print $cfile "$columnsAndTypes[$i]\t$columnsAndTypes[$i+1]\n";
+    }
+    close $cfile;
+}
+
+sub printSample {
+    # header line as a comment
+    print "#";
+    for (my $i = 0; $i < scalar @columnsAndTypes; $i += 3) {
+        print " $columnsAndTypes[$i]";
+    }
+    print "\n";
+
+    # sample values
+    my $i;
+    for ($i = 0; $i < scalar @columnsAndTypes - 3; $i += 3) {
+        print " $columnsAndTypes[$i+2]";
+    }
+    # last column is the COMMENT and it needs to be quoted
+    print " '$columnsAndTypes[$i+2]'";
+    print "\n";
+
+    exit 0;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+ftpsrequest - use ftcreate to create a postage stamp request fits table from a textual description
+
+=head1 SYNOPSIS
+
+  ftpsrequest --req_name <request name> --input <input text file> --output <output file name>
+
+    Options:
+        --req_name      REQ_NAME value for the fits header
+        --input         name of ASCII file giving the table values
+        --output        name of output fits table
+        --email         email address of user (optional)
+        --clobber       overwrite existing output file
+        --help          print usage message
+        --print-sample  print out an sample input text file
+
+=head1 NOTE
+
+  In order for ftpsrequest to succeed the ftools program ftcreate must be found in the users path.
+
+  FTOOLS are described at http://heasarc.gsfc.nasa.gov/ftools
+
+=cut
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psgetcalibinfo
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psgetcalibinfo	(revision 36680)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psgetcalibinfo	(revision 36680)
@@ -0,0 +1,107 @@
+#!/bin/env perl
+
+# pstamp_get_calib_info.pl
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use Carp;
+
+use IPC::Cmd 0.36 qw( can_run run );
+#use File::Temp qw( tempfile );
+#use File::Copy;
+#use File::Basename qw(dirname);
+
+use PS::IPP::Metadata::Config;
+# use PS::IPP::Metadata::Stats;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config qw( :standard );
+#use PS::IPP::PStamp::RequestFile qw( :standard );
+#use PS::IPP::PStamp::Job qw( :standard );
+
+my ( $cam_id, $output, $dbname, $verbose, $save_temps);
+
+GetOptions(
+           'cam_id=s'       => \$cam_id,
+           'output=s'       => \$output,
+	   'dbname=s'       => \$dbname,
+	   'verbose'        => \$verbose,
+	   'save-temps'     => \$save_temps,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+
+die "usage: --cam_id <cam_id> --output <output file name> [--dbname dbname --verbose]\n"
+    if !$cam_id or !$output;
+
+my $ipprc = PS::IPP::Config->new(); # IPP Configuration
+
+my $missing_tools;
+
+my $releasetool = can_run('releasetool') or (warn "Can't find releasetool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $relexp;
+{
+    my $command = "$releasetool -listrelexp -state calibrated -priority_order -limit 1 -cam_id $cam_id";
+    $command   .= " -dbname $dbname" if $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $data = join "", @$stdout_buf;
+    if ($data) {
+        # print STDERR $data if $verbose;
+
+        my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc");
+
+        # no need to use parse_md_fast here
+        my $results = parse_md_list($metadata);
+        if (scalar @$results != 1) {
+            print STDERR "get_job_params: failed to parse_md_list\n";
+            exit 0;
+        }
+
+        $relexp = $results->[0];
+    }
+}
+
+
+my $OUT;
+if ($output eq '-') {
+    $OUT = *STDOUT;
+} else {
+    open $OUT, ">$output" or die "failed to open $output for writing\n";
+}
+
+if (!$relexp) {
+    print STDERR "Failed to find calibration information for camRun: $cam_id\n";
+    print $OUT "DVOCALIB     STR         F # exposure not calibrated with DVO\n";
+    exit 0;
+}
+
+my $ubercal_exp = ($relexp->{flags} & 0x200) ? "T # exposure has ubercal zero point" : "F # exposure does not have ubercal zero point";
+
+print $OUT "DVOCALIB     STR         T # exposure calibrated with DVO\n";
+print $OUT "ZPCALIB      F32         $relexp->{zpt_obs} # calibrated zero point\n";
+print $OUT "ZPCALERR     F32         $relexp->{zpt_stdev} # calibrated zero point error - 0 for ubercal exposure\n";
+print $OUT "MCAL         F32         $relexp->{mcal} # zero point offset due to clouds\n";
+print $OUT "UCALEXP      STR         $ubercal_exp\n";
+print $OUT "DVOFLAGS     U32         $relexp->{flags} # dvo flags\n";
+print $OUT "UCALDIST     S32         $relexp->{ubercal_dist} # distance to ubercal image\n";
+print $OUT "DVODB        STR         $relexp->{dvodb} # dvo database used for calibration \n";
+print $OUT "UCALFILE     STR         $relexp->{ubercal_file} # ubercal file used for calibration\n";
+print $OUT "CALDATE      STR         $relexp->{time_stamp} # time of calibration\n";
+print $OUT "PSREL        STR         $relexp->{release_name} # PS release name\n";
+
+close $OUT or die "failed to close $output\n";
+
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq	(revision 36680)
@@ -39,8 +39,9 @@
 my $job_type     = 'stamp';
 my $req_type     = 'bycoord';
-my $stage        = 'chip';
+my $stage        = 'stack';
 my $option_mask;
 my $width        = $default_size;
 my $height       = $default_size;
+my $whole_file   = 0;
 my $project      = 'gpc1';
 my $coord_mask;
@@ -81,4 +82,5 @@
     'width=i'           => \$width,
     'height=i'          => \$height,
+    'whole-file'        => \$whole_file,
     'pixcenter'         => \$pixcenter,
     'arcseconds'        => \$arcseconds,
@@ -141,13 +143,22 @@
         pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 )
             if defined $x or defined $y;
-    } elsif (!$pixcenter) {
-        pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 )
-            unless (defined $ra and defined $dec);
-        # to simplify code we just use $x and $y from here
-        $x = $ra;
-        $y = $dec;
     } else {
-        pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 )
-                unless (defined $x and defined $y);
+        if ($whole_file) {
+            $pixcenter = 1;
+            $x = 0;
+            $y = 0;
+            $width = 0;
+            $height = 0;
+        }
+        if (!$pixcenter) {
+            pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 )
+                unless (defined $ra and defined $dec);
+            # to simplify code we just use $x and $y from here
+            $x = $ra;
+            $y = $dec;
+        } else {
+            pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 )
+                    unless (defined $x and defined $y);
+        }
     }
 } else {
@@ -177,5 +188,6 @@
 $id = 0 if !$id;
 
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack'
+    or $stage eq 'stack_summary') {
     die "$stage is not a valid value for stage\n";
 }
@@ -208,4 +220,5 @@
         $option_mask |= $PSTAMP_SELECT_PSF      if $psf;
         $option_mask |= $PSTAMP_SELECT_BACKMDL  if $backmdl;
+        $option_mask |= $PSTAMP_SELECT_UNCOMPRESSED if $uncompressed;
         $option_mask |= $PSTAMP_SELECT_UNCONV   if $unconvolved;
         $option_mask |= $PSTAMP_USE_IMFILE_ID   if $use_imfile_id;
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl	(revision 36680)
@@ -27,5 +27,5 @@
 my $IPP_DIFF_MODE_STACK_STACK = 4;
 
-my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
+my ($dep_id, $stage, $stage_id, $component, $imagedb, $label, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
 my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
 
@@ -36,5 +36,6 @@
     'component=s'   =>  \$component,
     'imagedb=s'     =>  \$imagedb,      # dbname for images lookups.
-    'rlabel=s'      =>  \$rlabel,
+    'label=s'       =>  \$label,        # request's label
+    'rlabel=s'      =>  \$rlabel,       # pstampDependent.rlabel (deprecated)
     'need_magic'    =>  \$need_magic,
     'fault_count=i' =>  \$fault_count,
@@ -73,4 +74,17 @@
 }
 
+if ($label) {
+    # rlabel is deprecated. Use one based on the supplied label parameter which is the current label 
+    # for the request, which may be different than the one given to the dependent when the job was parsed.
+    # XXX: having the convention that update label is 'ps_ud_' . $label of request embedded here 
+    # (and formerly in pstampparse.pl) is not particularly clean but it's simple.
+    my $new_rlabel = 'ps_ud_' . $label;
+    if ($new_rlabel ne $rlabel) {
+        print "Notice: using $new_rlabel instead of $rlabel for update label.\n";
+        $rlabel = $new_rlabel;
+    }
+}
+
+
 if (!$ps_dbserver) {
     $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
@@ -304,4 +318,20 @@
             }
             $queued_update = 1;
+        } elsif ($chip->{state} eq 'cleaned' and $chip->{data_state} eq 'update') {
+            # we've had a number of runs in this limbo state
+
+            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+            $queued_update = 1;
         } elsif ($chip->{fault}) {
             $fault_count++;
@@ -438,4 +468,5 @@
             $chips_ready = 0;
             $chip->{fault} = $chip->{chip_fault};
+            $chip->{data_group} = $chip->{chip_data_group};
             push @chipsToUpdate, $chip;
         } else {
@@ -459,4 +490,18 @@
             print "skipping $command\n";
         }
+    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
+        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+
     } elsif (scalar @chipsToUpdate > 0) {
         my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic);
@@ -528,4 +573,8 @@
             }
             return $warp_status;
+        } elsif ($warp->{quality} != 0) {
+            print STDERR "input warp has quality error\n";
+            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
+            return $PSTAMP_GONE;
         }
         # warps are ready fall through and queue the diff update
@@ -575,10 +624,10 @@
     } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
         # check the state of the input stack
-        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
+        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
         my $stack1 = runToolAndParseExpectOne($command, $verbose);
-        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
+        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
 
         if ($stack1->{state} ne 'full') {
-            print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+            print STDERR "input stack $skycell->{stack1} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
             faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
             return $PSTAMP_GONE;
@@ -590,5 +639,5 @@
 
         if ($stack2->{state} ne 'full') {
-            print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+            print STDERR "template stack $skycell->{stack2} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
             faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
             return $PSTAMP_GONE;
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_cleanup.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_cleanup.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_cleanup.pl	(revision 36680)
@@ -80,6 +80,8 @@
 }
 
+if (0) {
 my_die("Cleanup not yet supported for reqType: $reqType", $req_id, $PS_EXIT_UNKNOWN_ERROR)
     if ($reqType ne "pstamp") and ($reqType ne "NULL") and ($reqType ne "dquery");
+}
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_get_image_job.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_get_image_job.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_get_image_job.pl	(revision 36680)
@@ -65,5 +65,7 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $data = $mdcParser->parse(join "", (<INPUT>)) or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+my $data = $mdcParser->parse(join "", (<INPUT>)) 
+    or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+
 my $components = parse_md_list($data);
 my $n = scalar @$components;
@@ -83,5 +85,4 @@
     print STDERR "stage_id is $stage_id\n";
     print STDERR "path_base is $path_base\n";
-    print STDERR "path_base is $path_base\n";
     print STDERR "CAMERA is $camera\n";
     print STDERR "magicked is " . (defined $magicked ? $magicked : "undefined") . "\n";
@@ -89,5 +90,5 @@
 
 if (!$camera or !$path_base or !$component or !$stage_id or !$stage) {
-       my_die("failed to parse params from: $params_file", $PS_EXIT_UNKNOWN_ERROR);
+       my_die("One or more parameters are missing in: $params_file", $PS_EXIT_UNKNOWN_ERROR);
 }
 
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl	(revision 36680)
@@ -45,4 +45,5 @@
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
+my $fhead = can_run('fhead')  or (warn "Can't find fhead"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -56,22 +57,62 @@
     # Note that if it's a pstamp request then REQ_NAME should be defined.
     # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
-    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    # my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    my $command = "$fhead -x 0 $tmp_req_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-if (0) {
-    # stoopid fields doesn't set exit status to zero when it works
     unless ($success) {
         print STDERR @$stderr_buf;
         exit $error_code >> 8;
     }
-}
     my $output = join "", @$stdout_buf;
-    (undef, $extname, $extver, $req_name) = split " ", $output;
+
+
+    my $makehash = 0;
+    my %hash;
+    foreach my $line (split "\n", $output) {
+        chomp $line;
+        # split lines inte left and right using equals sign. left is the keyword
+        my ($key, $right) = split "=", $line;
+            # skip if there was no '='
+        next if !$right;
+
+        $key =~ s/ //g;
+
+        # separate value from comment
+        my ($value, $comment) = split "/", $right;
+        # remove ' and space characters from key and value
+        $value =~ s/'//g;
+        $value =~ s/ //g;
+
+        #    print "$key $value\n";
+
+        # extract the values that we are looking for
+        $extname  = $value if ($key eq "EXTNAME");
+        $extver   = $value if ($key eq "EXTVER");
+        $req_name = $value if ($key eq "REQ_NAME");
+        $req_name = $value if ($key eq "QUERY_ID");
+
+        # optionally build hash. Duplicate keys get the last value seen
+        if ($makehash) {
+            $hash{$key} = $value;
+        }
+    }
+
+    # (undef, $extname, $extver, $req_name) = split " ", $output;
+
     if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
         print STDERR "invalid request file\n";
+        print "invalid request file\n";
         exit $PS_EXIT_DATA_ERROR;
     }
+
     if (!defined $req_name) {
         print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
+        print "invalid request file no REQ_NAME or QUERY_ID\n";
+        exit $PS_EXIT_DATA_ERROR;
+    }
+    if (!defined $extver) {
+        print STDERR "invalid request file no EXTVER found\n";
+        print "invalid request file no EXTVER found\n";
         exit $PS_EXIT_DATA_ERROR;
     }
@@ -119,4 +160,5 @@
 
 exit 0;
+
 # Ask the database for the next web request number
 sub get_webreq_num
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl	(revision 36680)
@@ -57,8 +57,6 @@
 my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
 
-$options = 1 if !$options;
-
-# We don't need to muggle anymore. Ignore those options
-$options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
+# ppstamp requires an input file
+$options = $PSTAMP_SELECT_IMAGE if !$options;
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
@@ -78,7 +76,6 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
 my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
-my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
-my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
 
@@ -96,18 +93,25 @@
     my $argString;
     $argString = $params->{job_args};
+    
+    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
     my $stage = $params->{stage};
-
-    # XXX: should we do any other sanity checking?
-    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
-
-    my $nan_masked = 1;
-    my $muggle = 0;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
-        # Attempt to find or create a muggle image
-        $nan_masked = 0;
-        $muggle = 1;
-    }
+    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
+
+    if ($stage eq 'stack_summary') {
+
+        # remove options not supported by stack summary
+        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE 
+            | $PSTAMP_RESTORE_BACKGROUND);
+
+        # stackSummary outputs do not follow the usual IPP conventions for file names.
+        # The parser (actually Job.pm) has deferred handling this until here
+        update_stack_summary_filenames($params);
+
+    } elsif ($stage ne 'stack') {
+        # ignore options only supported by stack and stack_summary
+        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+    }
+   
 
     if ($stage eq "raw") {
@@ -122,5 +126,5 @@
     my @file_list = ($params->{image});
     
-    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
+    if ($options & $PSTAMP_SELECT_MASK) {
         $mask = $params->{mask};
         $fileArgs .= " -mask $mask";
@@ -162,101 +166,57 @@
     # find our output directory
     my $outdir = dirname($outputBase);
-    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
-    
-    if ($muggle) {
-        # first see if the original uncensored images are around
-        if (check_for_backups($params, \$fileArgs)) {
-            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
-            print "Making stamps from backup images\n";
-        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
-            # user required uncensored but since stage isn't chip we can't rebuild them
-            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
-            # we can only restore pixels for chip stage images if the data has been updated.
-            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
-            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
-            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
-            # make stamps from uncensored images
-            $muggle = 0;
+
+    my ($calib_fd, $calibfile);
+    if ($stage eq 'chip' or $stage eq 'warp') {
+        my $cam_id = $params->{cam_id};
+        if (!$cam_id) {
+            carp "no cam_id found in job params\n";
+            exit $PS_EXIT_PROG_ERROR;
+        }
+        ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps);
+        close $calib_fd;
+
+        my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        my $exitStatus;
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
         } else {
-            # Try and replace the streaks from the recovery images
-
-            my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
-            $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
-            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
-            # find the "directory" of the input path_base
-            my $inputdir = dirname($image);
-            my $base = basename($image);
-            $tmpImage = "$tmproot/$base";
-
-            @file_list = ();
-
-            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
-            # stage work. Import the rule here.
-            my $recImage = "$inputdir/REC_$base";
-            my $newFileArgs = " -file $tmpImage";
-            $muggle_command .= " -image $image -recimage $recImage";
-            push @file_list, $recImage;
-
-            if ($mask) {
-                $base = basename($mask);
-                $tmpMask = "$tmproot/$base";
-                $newFileArgs .= " -mask $tmpMask";
-                my $recMask = "$inputdir/REC_$base";
-                $muggle_command .= " -mask $mask -recmask $recMask";
-                push @file_list, $recMask;
-            }
-
-            if ($variance) {
-                $base = basename($variance);
-                $tmpVariance = "$tmproot/$base";
-                $newFileArgs .= " -variance $tmpVariance";
-                my $recVariance = "$inputdir/REC_$base";
-                $muggle_command .= " -weight $variance -recweight $recVariance";
-                push @file_list, $recVariance;
-            }
-            if (check_files(0, @file_list)) {
-                # recovery files exist and are accessible restore the excised pixels
-
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $muggle_command, verbose => $verbose);
-                unless ($success) {
-                    my $exitStatus = WEXITSTATUS($error_code);
-                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
-                }
-
-                # set the ppstamp file arguments
-                $fileArgs = $newFileArgs;
-            } else {
-                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
-                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
-                }
-                # just make stamps from the censored images
-                print "Unable to restore uncensored images, will extract stamps from censored images\n";
-                # these files won't be used so zap them
-                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
-            }
-        }
-    }
-
-    # MAGIC IS DEAD
-    $nan_masked = 0;
-
-    my $command = "$ppstamp $outputBase $argString $fileArgs";
-    $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
-    $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
-    $command .= " -stage $stage";
-    $command .= " -censor_masked" if $nan_masked;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+            print STDERR "psgetcalibinfo failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        exit $exitStatus if $exitStatus;
+
+        if (-s $calibfile == 0) {
+            print "no calibration information found for $cam_id\n";
+            $calibfile = undef;
+        }
+    }
+
+    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
+    # otherwise we make copies of the input files to the outputs
+    my $use_ppstamp = ($stage ne 'stack_summary');
 
     my $exitStatus;
-    if (WIFEXITED($error_code)) {
-        $exitStatus = WEXITSTATUS($error_code);
-    } else {
-        print STDERR "ppstamp failed error_code: $error_code\n";
-        $exitStatus = $PS_EXIT_SYS_ERROR;
+    if ($use_ppstamp) {
+        my $command = "$ppstamp $outputBase $argString $fileArgs";
+        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
+        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
+        $command .= " -stage $stage";
+        $command .= " -forheader $calibfile" if $calibfile;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (WIFEXITED($error_code)) {
+            $exitStatus = WEXITSTATUS($error_code);
+        } else {
+            print STDERR "ppstamp failed error_code: $error_code\n";
+            $exitStatus = $PS_EXIT_SYS_ERROR;
+        }
+        # XXX: if stage is stack deal with EXP and NUM images if selected
+    } else {
+        $exitStatus = justCopyFiles($outputBase, \$options, $params);
     }
 
@@ -270,11 +230,19 @@
 
         # Note: we are assuming the contents of the PSTAMP filerules here.
-        my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits", 
-                           $PSTAMP_SELECT_MASK     => "mk.fits",
-                           $PSTAMP_SELECT_VARIANCE => "wt.fits",
-                           $PSTAMP_SELECT_SOURCES  => "cmf",
-                           $PSTAMP_SELECT_JPEG     => "jpg");
-
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
+        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits', 
+                           $PSTAMP_SELECT_MASK     => 'mk.fits',
+                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
+                           $PSTAMP_SELECT_SOURCES  => 'cmf',
+                           $PSTAMP_SELECT_JPEG     => 'jpg',
+                           $PSTAMP_SELECT_EXP      => 'exp.fits',
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
+
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -298,6 +266,6 @@
         close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP || $exitStatus == $PSTAMP_NO_VALID_PIXELS) {
+        $jobStatus = $exitStatus;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -419,4 +387,7 @@
         my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $filter = $params->{filter};
+        $filter = ' ' if !$filter;
+	$filter = substr $filter, 0, 1;
         my $cmf_file;
 #        if ($stage ne 'chip') {
@@ -431,5 +402,8 @@
             if (!$therest or !$rownum or !$jobnum);
 
-        my $prefix = "${rownum}_${jobnum}_";
+        # XXX: Here we are assuming the form of the output file names
+	# if we change this in pstampparse we'll need to remember ....
+	# (the last time I forgot)
+        my $prefix = "${rownum}_${jobnum}_${filter}_";
 
         if ($cmf_file) {
@@ -523,38 +497,4 @@
 }
 
-sub check_for_backups {
-    my $params = shift;
-    my $r_fileArgs = shift;
-
-    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
-    my $results = runToolAndParse($command, $verbose);
-    my $dsComponent  = $results->[0];
-
-    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
-
-        print "magicDSFile state is full. Backup images should exist\n";
-
-        # replace the file names with the backup paths
-        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
-        my @args = split " ", $$r_fileArgs;
-
-        my $newFileArgs;
-        while (@args) {
-            my $a = shift @args;
-            my $f = shift @args;
-            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
-
-            my $backup_image_name = $ipprc->destreaked_filename($f);
-            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
-            $newFileArgs .= " $a $backup_image_name";
-        }
-        $$r_fileArgs = $newFileArgs;
-
-        return 1;
-    }
-
-    return 0;
-}
-
 my $neb;
 sub storage_object_exists
@@ -630,4 +570,99 @@
 }
 
+# stack_summary stage does not currently use proper file rules.
+# The parser defers handlint this to us..
+
+sub update_stack_summary_filenames {
+    my $params  = shift;
+
+    my $path_base = $params->{path_base};
+
+    $params->{image}  = $path_base . ".image.b1.fits";
+    $params->{mask}   = $path_base . ".mask.b1.fits";
+    $params->{weight} = $path_base . ".variance.b1.fits";
+    $params->{jpeg}   = $path_base . ".image.0.b1.jpeg";
+    $params->{exp}    = $path_base . ".exp.b1.fits";
+    $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
+}
+
+sub myCopy {
+    my ($dest, $src, $type, $dieOnFail) = @_;
+
+    my $result;
+    my $resolved = $ipprc->file_resolve($src);
+    if ($resolved and $ipprc->file_exists($resolved)) {
+        print "Copying $src to $dest\n" if $verbose;;
+        $result = copy($resolved, $dest);
+    } else {
+        my $msg = "Specified source $type image $src not found.";
+        if ($dieOnFail) {
+            $msg .= "\n";
+        } else {
+            $msg .= " ignoring\n";
+        }
+        carp $msg;
+        $result = 0;
+    }
+    
+    if (!$result and $dieOnFail) {
+        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
+    }
+    return $result;
+}
+
+sub justCopyFiles {
+    my ($outputBase, $r_options, $params) = @_;
+
+    print "Just copying files for $job_id.\n";
+
+    my $options = $$r_options;
+    if ($options & $PSTAMP_SELECT_IMAGE) {
+        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
+    }
+    if ($options & $PSTAMP_SELECT_MASK) {
+        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
+            $options &= ~$PSTAMP_SELECT_MASK;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_VARIANCE) {
+        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
+            $options =  ~$PSTAMP_SELECT_VARIANCE;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_JPEG) {
+        if (!myCopy("$outputBase.jpg", $params->{jpeg}, 'jpeg', 0)) {
+            $options &= ~$PSTAMP_SELECT_JPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXP) {
+        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXP;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUM) {
+        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
+        }
+    }
+
+    $$r_options = $options;
+
+    print "Done with copy.\n";
+
+    return 0;
+}
+
 sub my_die
 {
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_queue_cleanup.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_queue_cleanup.pl	(revision 36680)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_queue_cleanup.pl	(revision 36680)
@@ -0,0 +1,80 @@
+#!/bin/env perl
+#
+# run pstamptool to queue requests older than preserve-days to be cleaned
+
+use strict;
+use warnings;
+
+use PS::IPP::Config 1.01 qw( :standard );
+use IPC::Cmd 0.36 qw( can_run run );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+my $save_temps;
+my $preserve_days;
+my $dbname;
+my $dbserver;
+
+GetOptions(
+    'preserve-days=s'       => \$preserve_days,     # clean up requests that stopped this many days ago
+    'verbose'               => \$verbose,
+    'save-temps'            => \$save_temps,
+    'dbserver=s'            => \$dbserver,
+    'dbname=s'              => \$dbname,
+) or pod2usage( 2 );
+
+my $missing_tools;
+my $pstamptool   = can_run('pstamptool') 
+    or (warn "Can't find pstamptool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $ipprc = PS::IPP::Config->new(); # IPP Configuration
+
+if (!$dbserver) {
+    $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+    $pstamptool .= " -dbserver $dbserver" if $dbserver;
+}
+
+if (!$dbname) {
+    $dbname =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBNAME');
+    $pstamptool .= " -dbname $dbname" if $dbname;
+}
+
+
+
+if (!$preserve_days) {
+    $preserve_days = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_PRESERVE_DAYS');
+    if (!$preserve_days) {
+        $preserve_days = 14;
+    }
+}
+
+if ($preserve_days < 0) {
+    warn("preserve-days cannot be negative\n" );
+    exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+my $ticks = time();
+$ticks -= 86400 * $preserve_days;
+
+my ($sec, $min, $hour, $mday, $month, $year) = gmtime($ticks);
+$year += 1900;
+$month += 1;
+
+my $timestamp_end = sprintf "%4d-%02d-%02dT%02d:%02d:%02d", $year, $month, $mday, $hour, $min, $sec;
+
+my $command = "$pstamptool -updatereq -set_state goto_cleaned -state stop -timestamp_end $timestamp_end";
+my  ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run(command => $command, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or 1);
+    warn("$command failed. exit status: $error_code");
+    exit $error_code;
+}
+
+
+exit 0;
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_save_server_status.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_save_server_status.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_save_server_status.pl	(revision 36680)
@@ -61,5 +61,6 @@
     $year, $month, $mday, $hour, $min, $sec;
 
-my $command = "pstamp_server_status > $file";
+my $command = "pstamp_server_status --workdir $pstamp_workdir > $file";
+#my $command = "pstamp_server_status > $file";
 my  ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run(command => $command, verbose => $verbose);
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_server_status
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_server_status	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_server_status	(revision 36680)
@@ -15,9 +15,10 @@
 
 
-my ($rundir, $verbose, $save_temps);
+my ($rundir, $workdir, $verbose, $save_temps);
 my $br = '<br />';
 
 GetOptions(
     'rundir=s'      => \$rundir,
+    'workdir=s'     => \$workdir,
     'verbose'       => \$verbose,
     'save-temps'    => \$save_temps,
@@ -155,4 +156,15 @@
     }
 }
+if ($workdir) {
+    # get the space available in the working directory file system
+    my $df_output = `df -h $workdir`;
+    my @lines = split "\n", $df_output;
+    foreach my $line (@lines) {
+        next unless $line =~ /data/;
+        my ($total, $used, $free, $percent, $part) = split " ", $line;
+        print "<br><b>Server Working Directory:</b>&nbsp;&nbsp;$percent full. $free free out of $total total.<br>\n";
+    }
+
+}
 
 # now run the script psstatus to check the status of running requests and finished requests
Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl
===================================================================
--- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl	(revision 35859)
+++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl	(revision 36680)
@@ -31,4 +31,5 @@
 my $no_update;
 my $dest_requires_magic;
+my $dump_params = 0;
 
 # set this to true to disable update processing
@@ -48,16 +49,19 @@
     'save-temps'=>  \$save_temps,
     'no-update' =>  \$no_update,
+    'dump-params' => \$dump_params,
 );
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76,5 +80,5 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116,14 +120,19 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175,17 +184,22 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212,26 +226,33 @@
 }
 
-my $nRows = scalar @$rows;
+my $nRows = $rows ? scalar @$rows : 0;
 print "\n$nRows rows read from request file\n";
 
+unless ($nRows) {
+    # pstamp_job_run was invoked so the request file must have contained a valid header
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
+    print STDERR "Invalid request file.\n";
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    exit 0;
+}
+
+
+# if label is for one of the high priority channels, watch for big requests
+my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
+
+# XXX: these should be in a configuration file somewhere not hard coded
+my $big_limit = 100;
+my $job_big_limit = 400;
+
+if ($watch_for_big_requests and $nRows > $big_limit) {
+    $label = change_to_lower_priority_label($label);
+    $watch_for_big_requests = 0;
+}
 
 my $num_jobs = 0;
-my $imageList;
-my $stage;
+
 foreach my $row (@$rows) {
-
-    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
-        # this is a big request and it came from the upload page and doesn't have a specific label assigned
-        # change it to the generic one that has lower with lower priority
-        $label = 'WEB.BIG';
-        print "\nChanging label for big WEB.UP request to $label\n";
-
-        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
 
     # validate the paramaters
@@ -246,12 +267,37 @@
 
     $num_jobs += processRow($action, $row);
+
+    # see whether number of jobs limit for high priority request was exceeded
+    if ($watch_for_big_requests and $num_jobs > $job_big_limit) {
+        $label = change_to_lower_priority_label($label);
+        $watch_for_big_requests = 0;
+    }
 }
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
 exit 0;
+
+# end of main function
+
+sub change_to_lower_priority_label {
+    my $label = shift;
+    my $old_label = $label;
+    $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+    print "\nChanging label for big $old_label request to $label\n";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    return $label;
+}
 
 sub checkRow {
@@ -267,4 +313,5 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
+	$rownum = 'NULL' if !defined $rownum;
         print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -272,5 +319,6 @@
     }
     my $job_type = $row->{JOB_TYPE};
-    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+    if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) {
+    	$job_type = 'NULL' if !defined $job_type;
         print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -281,4 +329,5 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+	$req_type = 'NULL' if !defined $req_type;
         print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -286,6 +335,9 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
+        # get_image jobs are quite expensive in terms of space so we are currently restricting them
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' 
+            or ($req_type eq 'byskycell' and $stage eq 'stack')
+            or ($stage eq 'stack_summary')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -295,5 +347,5 @@
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        if ($job_type eq 'get_image') {
+        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
             $row->{COMPONENT} = 'all';
         } else {
@@ -384,6 +436,6 @@
     }
 
-    if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
+    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
+        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -457,7 +509,7 @@
     my $start_locate = gettimeofday();
 
-    print "\nCalling new_locate_images for row: $rownum\n";
-
-    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+    print "\nCalling locate_images_for_row for row: $rownum\n";
+
+    my $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -521,5 +573,5 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -532,37 +584,7 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -581,5 +603,5 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
+    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -701,6 +723,20 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);
 
@@ -758,4 +794,11 @@
             print "$image->{image}\n";
             ++$firstRow->{job_num};
+            if ($dump_params) {
+                my $rownum = $firstRow->{ROWNUM};
+                my $jobnum = $firstRow->{job_num};
+                my $filter = substr $image->{filter}, 0, 1;
+                my $output_base = "${rownum}_${jobnum}_${filter}";
+                write_params($output_base, $image);
+            }
         }
     } elsif ($job_type eq "get_image") {
@@ -1163,5 +1206,5 @@
     }
     if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
-	($img_type eq "stack") or ($img_type eq "diff")) {
+	($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
 	return 1;
     } else {