Index: /branches/eam_branches/ipp-20100823/DataStore/lib/DataStore/Utils.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/DataStore/lib/DataStore/Utils.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/DataStore/lib/DataStore/Utils.pm	(revision 29515)
@@ -42,5 +42,5 @@
 $BYTE_FIELD = qr/^\d+$/;
 $MD5_FIELD = qr/^[0-9a-f]{32}$/;
-%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits );
+%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS );
 %KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST);
 %KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults);
Index: /branches/eam_branches/ipp-20100823/DataStore/scripts/dsgetfileset
===================================================================
--- /branches/eam_branches/ipp-20100823/DataStore/scripts/dsgetfileset	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/DataStore/scripts/dsgetfileset	(revision 29515)
@@ -17,5 +17,5 @@
 use File::Basename qw( basename );
 
-my ($uri, $outdir, $timeout, $skip_checks, $unpack, $no_proxy, $verbose);
+my ($uri, $outdir, $timeout, $skip_checks, $unpack, $first_file, $no_proxy, $verbose);
 
 GetOptions(
@@ -25,4 +25,5 @@
     'skip-checks'       => \$skip_checks,
     'unpack'            => \$unpack,
+    'first-file=s'      => \$first_file,
     'no-proxy'          => \$no_proxy,
     'verbose|v'         => \$verbose,
@@ -75,4 +76,8 @@
     }
 
+    if ($first_file) {
+        next if $fs->fileid ne $first_file;
+        $first_file = undef;
+    }
     my $uri = $fs->uri;
     my $base = basename($uri);
@@ -153,5 +158,9 @@
 =item * --unpack
 
-Uncompress fits files if compressed
+Uncompress fits files if compressed.
+
+=item * --first-file <fileid>
+
+Skip files in fileset list prior to the given fileid.
 
 Optional.
Index: /branches/eam_branches/ipp-20100823/DataStoreServer/scripts/dsreg
===================================================================
--- /branches/eam_branches/ipp-20100823/DataStoreServer/scripts/dsreg	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/DataStoreServer/scripts/dsreg	(revision 29515)
@@ -42,4 +42,5 @@
 
 my $dbname;         # Database name
+my $dbserver;       # Database host
 my $dsroot;         # root directory of the data store
 
@@ -82,8 +83,14 @@
         'ps7=s'         =>      \$ps7,
         'dbname=s'      =>      \$dbname,
+        'dbserver=s'    =>      \$dbserver,
+        'ds_dbname=s'   =>      \$dbname,       # some scripts use these names
+        'ds_dbserver=s' =>      \$dbserver,
         'dsroot=s'      =>      \$dsroot,
         'verbose'       =>      \$verbose,
 ) or pod2usage(2);
 
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+
+# collect all command line argument errors and only fail once below.
 my $err = "";
 
@@ -121,8 +128,7 @@
     $dsroot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
     if (!$dsroot) {
-        die("Data Store root directory not set");
-    }
-}
-
+        &my_die("Data Store root directory not set", $PS_EXIT_CONFIG_ERROR);
+    }
+}
 
 if (!stat("$dsroot/index.txt")) {
@@ -131,12 +137,11 @@
 
 # bail if any errors above
-show_usage($err)
-    if ($err);
-
-show_usage("Invalid product '$product'.")
+show_usage($err) if ($err);
+
+show_usage("Invalid data store product '$product'.")
     unless defined stat("$dsroot/$product/index.txt");
 
 $dbname      = metadataLookupStr($siteConfig, 'DS_DBNAME') unless defined $dbname;
-my $dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER');
+$dbserver    = metadataLookupStr($siteConfig, 'DS_DBSERVER') unless defined $dbserver;
 my $dbuser   = metadataLookupStr($siteConfig, 'DS_DBUSER');
 my $dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD');
@@ -148,5 +153,5 @@
 
 my $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
-        or die "Cannot connect to DB server\n";
+        or &my_die("Cannot connect to DB server\n", $PS_EXIT_SYS_ERROR);
 
 print STDERR "dbh: $dbh\n" if $verbose;
@@ -161,5 +166,9 @@
 
     my $stmt = $dbh->prepare("SELECT prod_id, last_fs FROM dsProduct WHERE prod_name = ?");
-    $stmt->execute($product);
+    my $execute_status = $stmt->execute($product);
+    if (!$execute_status) {
+        print STDERR "failed to execute product lookup\n";
+        exit 2;
+    }
 
     my $prod_row = $stmt->fetchrow_hashref();
@@ -228,5 +237,5 @@
         my $rc;
         if (($rc = system "rm -r $fileset_dir")) {
-            die("failed to remove $fileset_dir error code: $rc");
+            &my_die("failed to remove $fileset_dir error code: $rc", $PS_EXIT_UNKNOWN_ERROR);
         }
     } else  {
@@ -252,5 +261,5 @@
 
     if (!$prod_id) {
-        die("product $product not found");
+        &my_die("product $product not found", $PS_EXIT_UNKNOWN_ERROR);
     }
     my $count = $dbh->do("SELECT fileset_id FROM dsFileset WHERE fileset_name = ?" .
@@ -268,8 +277,8 @@
     if (! -e $fileset_dir) {
         if (!mkdir $fileset_dir) {
-            die("failed trying to create fileset directory $fileset_dir");
+            &my_die("failed trying to create fileset directory $fileset_dir", $PS_EXIT_SYS_ERROR);
         }
 	if (! -d $fileset_dir ) {
-            die("cannot access just created fileset directory $fileset_dir");
+            &my_die("cannot access just created fileset directory $fileset_dir", $PS_EXIT_SYS_ERROR);
 	}
     }
@@ -281,5 +290,5 @@
             $in = *STDIN;
         } else {
-            open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
+            open $in, "<$filelist" or &my_die("cannot open filelist file $filelist\n", $PS_EXIT_UNKNOWN_ERROR);
         }
 
@@ -296,5 +305,5 @@
             my @fields = split /\|/;
             if (@fields < 4) {
-                die "malformed input line at $filelist line $lineno: $_\n";
+                &my_die("malformed input line at $filelist line $lineno: $_\n", $PS_EXIT_PROG_ERROR);
             }
             $filename = shift @fields;
@@ -320,5 +329,5 @@
         }
 
-        die("empty filelist\n") if (@files == 0);
+        &my_die("empty filelist\n", $PS_EXIT_PROG_ERROR) if (@files == 0);
 
         # Prepare the fileset directory
@@ -350,5 +359,5 @@
                     } else {
                         if (!copy($src, $dest)) {
-                            die("copy($src, $dest) failed");
+                            &my_die("copy($src, $dest) failed", $PS_EXIT_UNKNOWN_ERROR);
                         }
                     }
@@ -359,5 +368,5 @@
 
                 if (!$no_cleanup && system "rm -r $fileset_dir") {
-                    die("failed to remove $fileset_dir");
+                    &my_die("failed to remove $fileset_dir", $PS_EXIT_UNKNOWN_ERROR);
                 }
                 exit $PS_EXIT_UNKNOWN_ERROR;
@@ -370,10 +379,10 @@
             my $path = "$fileset_dir/$filename";
             if (! -e $path ) {
-                die "file $path not found";
+                &my_die("file $path not found", , $PS_EXIT_UNKNOWN_ERROR);
             }
             # get the size of the file
             my @finfo = stat($path);
             unless (@finfo) {
-                die ("can't stat $path");
+                &my_die ("can't stat $path", $PS_EXIT_UNKNOWN_ERROR);
             }
             # if size was supplied make sure that it matches the actual
@@ -382,6 +391,6 @@
                 my $current_size = $finfo[7];
                 if ($file->{bytes} != $current_size) {
-                    die "size on disk: $current_size does not match supplied"
-                    . " size: $file->{bytes} for $path";
+                    &my_die("size on disk: $current_size does not match supplied"
+                    . " size: $file->{bytes} for $path", $PS_EXIT_PROG_ERROR);
                 }
             } else {
@@ -391,5 +400,5 @@
                 # Get MD5 sum
                 $file->{md5sum} = file_md5_hex($path);
-                die("failed to compute valid md5sum for $path") if !$file->{md5sum};
+                &my_die("failed to compute valid md5sum for $path", $PS_EXIT_UNKNOWN_ERROR) if !$file->{md5sum};
             }
         }
@@ -401,5 +410,5 @@
         $dbh->disconnect();
         $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
-            or die "Cannot connect to server\n";
+            or &my_die("Cannot connect to server\n", $PS_EXIT_SYS_ERROR);
         print STDERR "ping failed new dbh: $dbh\n" if $verbose;
     }
@@ -415,5 +424,5 @@
                             $ps0, $ps1, $ps2, $ps3, $ps4, $ps5, $ps6, $ps7));
 
-        die("failed to insert $fileset") if (!defined($count) or ($count == 0E0));
+        &my_die("failed to insert $fileset", $PS_EXIT_UNKNOWN_ERROR) if (!defined($count) or ($count == 0E0));
 
         # I don't use last_insert_id() because I've seen some problems with it and
@@ -423,7 +432,7 @@
         my $nfs = @fsids;
         if (!$nfs) {
-            die("fileset '$fileset' missing even though we just added it");
+            &my_die("fileset '$fileset' missing even though we just added it", $PS_EXIT_UNKNOWN_ERROR);
         } elsif ($nfs > 1) {
-            die("Fileset '$fileset' already exists under $product.");
+            &my_die("Fileset '$fileset' already exists under $product.", $PS_EXIT_PROG_ERROR);
         }
         my $fileset_id = $fsids[0];
@@ -443,5 +452,5 @@
 
             if ($count == 0E0) {
-                die("failed to insert $filename into $fileset");
+                &my_die("failed to insert $filename into $fileset", $PS_EXIT_UNKNOWN_ERROR);
             }
         }
@@ -450,5 +459,5 @@
                                 . " WHERE prod_id = $prod_id");
         if ($count == 0E0) {
-            die("failed to update dsProduct $prod_id");
+            &my_die("failed to update dsProduct $prod_id", $PS_EXIT_UNKNOWN_ERROR);
         }
 
@@ -457,5 +466,5 @@
         # create the cgi script index.txt by making a copy of its parent's
         if (!copy("$dsroot/$product/index.txt", $index_script_name)) {
-            die("failed to copy index script to file set");
+            &my_die("failed to copy index script to file set", $PS_EXIT_UNKNOWN_ERROR);
         }
     };
@@ -531,2 +540,9 @@
     }
 }
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+    carp $msg;
+    exit $fault;
+}
Index: /branches/eam_branches/ipp-20100823/Nebulous/bin/neb-replicate
===================================================================
--- /branches/eam_branches/ipp-20100823/Nebulous/bin/neb-replicate	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Nebulous/bin/neb-replicate	(revision 29515)
@@ -64,5 +64,5 @@
 
 if ($set_copies) {
-    $neb->setxattr($key, "user.copies", $copies, "replace")
+    $neb->setxattr($key, "user.copies", $set_copies, "replace")
         or die $neb->err;
 }
Index: /branches/eam_branches/ipp-20100823/Ohana/src/addstar/include/addstar.h
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/addstar/include/addstar.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/addstar/include/addstar.h	(revision 29515)
@@ -121,4 +121,7 @@
 char    ZERO_POINT_OPTION[64];
 float	ZERO_POINT_OFFSET;
+float	ZERO_POINT_ERROR;
+float   ZPT_OBS_PHU;
+float   ZPT_ERR_PHU;
 
 // carries the mosaic into gstars
Index: /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/GetZeroPointExposure.c
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/GetZeroPointExposure.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/GetZeroPointExposure.c	(revision 29515)
@@ -17,7 +17,8 @@
 
 // PHU_HEADER: in this case, the zero point measured for the entire exposure and reported in
-// the PHU header is used to set the zero point offset for each chip.  Note that in this case,
-// ZERO_POINT_OFFSET in this funciton is actually set to the zero point, and adjusted to an
-// offset in ReadImageHeader based on the per-chip zero points in the photcode database
+// the PHU header is used to set the zero point offset for each chip.  
+// Here set ZPT_OBS_PHU and ZPT_ERR_PHU to the observed values from the header.
+// ZERO_POINT_OFFSET is calculated in ReadImageHeader based on the per-chip zero points in the
+// photcode database.
 
 int GetZeroPointExposure (Header **headers, HeaderSet *headerSets, off_t Nimages) {
@@ -26,4 +27,5 @@
     if (!strcasecmp(ZERO_POINT_OPTION, "NOMINAL")) {
 	ZERO_POINT_OFFSET = 0.0;
+	ZERO_POINT_ERROR = NAN;
 	return (TRUE);
     }
@@ -32,4 +34,5 @@
     if (!strcasecmp(ZERO_POINT_OPTION, "CHIP_HEADER")) {
 	ZERO_POINT_OFFSET = 0.0;
+	ZERO_POINT_ERROR = NAN;
 	return (TRUE);
     }
@@ -39,5 +42,5 @@
 
 	int i, Nzpt, Nmid, Nhead;
-	float *zpt, ZPT_OBS;
+	float *zpt, ZPT_OBS, ZPT_ERR;
 	PhotCode *photcode;
 	char photname[80];
@@ -53,4 +56,11 @@
 		fprintf (stderr, "zero point not supplied in header\n");
 		continue;
+	    }
+	    
+	    if (!gfits_scan (headers[Nhead], "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
+//		XXX should we emit this message? We currently aren't using this value so no
+//		fprintf (stderr, "zero point not supplied in header\n");
+//		continue;
+	        ZPT_ERR = NAN;
 	    }
 	    
@@ -78,4 +88,5 @@
 	    fprintf (stderr, "WARNING: zero point is not measured, no valid entries in headers\n");
 	    ZERO_POINT_OFFSET = 0.0;
+	    ZERO_POINT_ERROR = NAN;
 	    free (zpt);
 	    return (FALSE);
@@ -86,4 +97,6 @@
 	ZERO_POINT_OFFSET = (Nzpt % 2) ? zpt[Nmid] : 0.5*(zpt[Nmid] + zpt[Nmid-1]);
 	free (zpt);
+        // XXX: TODO calculate someting for ZERO_POINT_ERROR
+	ZERO_POINT_ERROR = NAN;
 	return (TRUE);
     }
@@ -92,16 +105,24 @@
     if (!strcasecmp(ZERO_POINT_OPTION, "PHU_HEADER")) {
 	int i, Nhead;
-	float ZPT_OBS;
 
 	ZERO_POINT_OFFSET = 0.0;
+	ZERO_POINT_ERROR = NAN;
 	for (i = 0; i < Nimages; i++) {
 	    if (strcmp(headerSets[i].exthead, "PHU")) continue;
 	    Nhead = headerSets[i].extnum_head;
 	    
+            float ZPT_OBS, ZPT_ERR;
 	    if (!gfits_scan (headers[Nhead], "ZPT_OBS", "%f", 1, &ZPT_OBS)) {
 		fprintf (stderr, "WARNING: zero point is not measured, no valid entries in headers\n");
 		return (FALSE);
 	    }
-	    ZERO_POINT_OFFSET = ZPT_OBS;
+	    if (!gfits_scan (headers[Nhead], "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
+		fprintf (stderr, "WARNING: zero point error is not measured\n");
+//		XXX: Do we want to require ZPT_ERR? for now just set it to zero and proceed.
+//		return (FALSE);
+		ZPT_ERR = NAN;
+	    }
+            ZPT_OBS_PHU = ZPT_OBS;
+            ZPT_ERR_PHU = ZPT_ERR;
 	    return (TRUE);
 	}
Index: /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/ReadImageHeader.c
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/ReadImageHeader.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/addstar/src/ReadImageHeader.c	(revision 29515)
@@ -220,6 +220,14 @@
           fprintf (stderr, "zero point not supplied in header\n");
           ZERO_POINT_OFFSET = 0.0;
+          ZERO_POINT_ERROR = NAN;
       } else {
           ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZPT_OBS;
+          float ZPT_ERR;
+          if (!gfits_scan (header, "ZPT_ERR", "%f", 1, &ZPT_ERR)) {
+              // XXX: do we want to print this message?
+              fprintf (stderr, "zero point error not supplied in header\n");
+              ZPT_ERR = NAN;
+          }
+          ZERO_POINT_ERROR = ZPT_ERR;
       }
   }
@@ -230,6 +238,8 @@
           fprintf (stderr, "photcode data not supplied for this chip\n");
           ZERO_POINT_OFFSET = 0.0;
+          ZERO_POINT_ERROR = NAN;
       } else {
-          ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZERO_POINT_OFFSET;
+          ZERO_POINT_OFFSET = 0.001*photcodeData[0].C - ZPT_OBS_PHU;
+          ZERO_POINT_ERROR =  ZPT_ERR_PHU;
       }
   }
@@ -237,4 +247,5 @@
   /* secz is in units milli-airmass */
   image[0].Mcal = ZERO_POINT_OFFSET;
+  image[0].dMcal = ZERO_POINT_ERROR;
   image[0].Xm   = NAN_S_SHORT;
   image[0].flags = 0;
Index: /branches/eam_branches/ipp-20100823/Ohana/src/dvomerge/src/merge_catalogs_old.c
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/dvomerge/src/merge_catalogs_old.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/dvomerge/src/merge_catalogs_old.c	(revision 29515)
@@ -317,4 +317,15 @@
     objID ++;
 
+    for (j = 0; j < NsecfiltOut; j++) {
+        int outputIndex = (Nave * NsecfiltOut) + j;
+	output[0].secfilt[outputIndex].M     = NAN;
+	output[0].secfilt[outputIndex].dM    = NAN;
+	output[0].secfilt[outputIndex].Xm    = NAN_S_SHORT;
+	output[0].secfilt[outputIndex].M_20  = NAN_S_SHORT;
+	output[0].secfilt[outputIndex].M_80  = NAN_S_SHORT;
+	output[0].secfilt[outputIndex].Ncode = 0;
+	output[0].secfilt[outputIndex].Nused = 0;
+    }
+
     for (j = 0; j < NsecfiltIn; j++) {
       int outputIndex;
@@ -333,12 +344,4 @@
 	output[0].secfilt[outputIndex].Ncode = input[0].secfilt[N*NsecfiltIn+j].Ncode;
 	output[0].secfilt[outputIndex].Nused = input[0].secfilt[N*NsecfiltIn+j].Nused;
-      } else {
-	output[0].secfilt[outputIndex].M     = NAN;
-	output[0].secfilt[outputIndex].dM    = NAN;
-	output[0].secfilt[outputIndex].Xm    = NAN_S_SHORT;
-	output[0].secfilt[outputIndex].M_20  = NAN_S_SHORT;
-	output[0].secfilt[outputIndex].M_80  = NAN_S_SHORT;
-	output[0].secfilt[outputIndex].Ncode = 0;
-	output[0].secfilt[outputIndex].Nused = 0;
       }
     }
Index: /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/dvo_photcode_ops.c
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/dvo_photcode_ops.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/dvo_photcode_ops.c	(revision 29515)
@@ -539,15 +539,24 @@
 // Create a map between the secfilt values from one photcode table to another
 int *GetSecFiltMap(PhotCodeData *output, PhotCodeData *input) {
-  int i;
+  int i, j;
   int NsecfiltIn  = input[0].Nsecfilt;
+  int NsecfiltOut = output[0].Nsecfilt;
   int *map;
   ALLOCATE(map, int, NsecfiltIn);
 
-  // loop over entries 
+  // loop over entries in the input table
   for (i = 0; i < NsecfiltIn; i++) {
     int code  = input[0].codeNsec[i];
-    int entry = output[0].hashcode[code];
+    int entry = -1;
+    // find the matching entry in the output table
+    for (j = 0; j < NsecfiltOut; j++) {
+      int outcode = output[0].codeNsec[j];
+      if (code == outcode) {
+	 entry = j;
+	 break;
+      }
+    }
     if (entry == -1) {
-      // entry is missing
+      // entry is missing fail (no printfs in this file)
       free(map);
       return(FALSE);
Index: /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/skyregion_ops.c
===================================================================
--- /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/skyregion_ops.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/Ohana/src/libdvo/src/skyregion_ops.c	(revision 29515)
@@ -195,4 +195,11 @@
     Dmin = MIN (Dmin, d);
     Dmax = MAX (Dmax, d);
+  }
+
+  if( Rmax > (Rmin+180.)) {
+    
+      double temp = Rmax;
+      Rmax = Rmin;
+      Rmin = temp;
   }
 
Index: /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 29515)
@@ -100,5 +100,5 @@
         my $x = $row->{CENTER_X};
         my $y = $row->{CENTER_Y};
-        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
         return $results;
     }
@@ -164,5 +164,5 @@
 
     my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
-        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
 
     return $results;
@@ -187,4 +187,6 @@
     my $data_group = shift;
     my $option_mask = shift;
+    my $mjd_min = shift;
+    my $mjd_max = shift;
     my $verbose  = shift;
 
@@ -287,4 +289,5 @@
             $choose_components = 1;
         }
+        $command .= " -template" if $inverse;
         $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id";
         $image_name  = "PPSUB.OUTPUT";
@@ -347,4 +350,7 @@
         $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
         $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
+    } else {
+        $command .= " -mjd_obs_begin $mjd_min" if $mjd_min;
+        $command .= " -mjd_obs_end $mjd_max" if $mjd_max;
     }
 
@@ -361,5 +367,5 @@
         # The image selectors are such that multiple runs my have be returned for the same exposure.
         # Return only the latest one.
-        $images = filterRuns($stage, $need_magic, $images, $verbose);
+        $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose);
     }
     if ($choose_components) {
@@ -379,5 +385,8 @@
 
         next if $image->{fault};
-        next if ($stage ne "raw") and $image->{quality};
+        if (($stage ne "raw") and $image->{quality}) {
+            print STDERR "Selected $stage image has bad quality. Skipping.\n";
+            next;
+        }
 
         if ($base_name) {
@@ -479,5 +488,5 @@
     my $listrun = 0;
     if ($byid) {
-        if ($skycell_id) {
+        if ($skycell_id and ($skycell_id ne 'all')) {
             $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id";
         } else {
@@ -634,4 +643,6 @@
     my $data_group = shift;
     my $option_mask = shift;
+    my $mjd_min = shift;
+    my $mjd_max = shift;
     my $verbose    = shift;
 
@@ -649,5 +660,5 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id},
                     $tess_id, $chip->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
 
                 next if !$these_results;
@@ -669,5 +680,5 @@
                 my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
                     $skycell->{tess_id}, $skycell->{component}, $need_magic, 
-                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+                    $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose);
 
                 next if !$these_results;
@@ -740,4 +751,8 @@
         my $astrom_resolved = $ipprc->file_resolve($astrom);
         next if !$astrom_resolved;
+        if (! -e $astrom_resolved) {
+            print "$astrom_resolved not found skipping\n";
+            next;
+        }
 
         my $start_dvo = gettimeofday();
@@ -1001,4 +1016,12 @@
 
     my $ticks = ($mjd - 40587.0) * 86400;
+
+    # CONVERT from TAI to UTC
+    # XXX: do this correctly
+    if ($mjd >= 54832) {
+        $ticks += -34;
+    } else {
+        $ticks += -33;
+    }
 
     my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
@@ -1202,9 +1225,5 @@
                 my $images = runToolAndParse($command, $verbose);
                 if (!defined $images) {
-                    print "No components containing coordinates found for ${stage}_id $stage_id\n";
-                    foreach my $row (@$rowList) {
-                        # XXX: This doesn't seem like the correct error code?
-                        $row->{error_code} = $PSTAMP_NO_OVERLAP;
-                    }
+                    print "No  $c found for $stage ${stage}_id $stage_id\n";
                     next;
                 }
@@ -1235,4 +1254,5 @@
     my $need_magic = shift;
     my $inputs     = shift;
+    my $inverse     = shift;
     my $verbose    = shift;
 
@@ -1254,5 +1274,14 @@
     my $printed = 0;
     foreach my $input (@$inputs) {
-        my $exp_id = $input->{exp_id};
+        my $exp_id;
+        if ($stage ne "diff") {
+            $exp_id = $input->{exp_id};
+        } elsif (!$inverse) {
+            $exp_id = $input->{exp_id_1};
+            $input->{exp_id} = $exp_id;
+        } else {
+            $exp_id = $input->{exp_id_2};
+            $input->{exp_id} = $exp_id;
+        }
         my $run_id = $input->{$id_name};
         my $magicked = $input->{magicked};  # this will be either stageRun.magicked or stage%file.magicked
@@ -1260,4 +1289,5 @@
 
         # can't process run in these states
+        # XXX: also goto_purged or goto_scrubbed or drop
         if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) {
             print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n";
Index: /branches/eam_branches/ipp-20100823/dbconfig/background.md
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/background.md	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/background.md	(revision 29515)
@@ -21,4 +21,5 @@
     class_id            STR     64
     path_base           STR     255
+    data_state          STR     64
     magicked            S64     0
     dtime_script        F32     0.0
@@ -57,4 +58,5 @@
     skycell_id          STR     64
     path_base           STR     255
+    data_state          STR     64
     magicked            S64     0
     dtime_script        F32     0.0
Index: /branches/eam_branches/ipp-20100823/dbconfig/changes.txt
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/changes.txt	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/changes.txt	(revision 29515)
@@ -1945,2 +1945,22 @@
 ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell);
 ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell);
+
+ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;
+
+-- Version 1.1.65
+-- changes to support cleanup of the background preserved images
+ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
+ALTER TABLE chipBackgroundImfile ADD KEY(data_state);
+ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
+ALTER TABLE warpBackgroundSkyfile  ADD KEY(data_state);
+
+ALTER TABLE magicMask ADD COLUMN path_base VARCHAR(255) AFTER uri;
+
+-- diff_id was removed from magicInputSkyfile ages ago, but
+-- pxadmin_create_tables.sql still had it this command didn't work due to
+-- a mysql error.
+-- ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
+UPDATE dbversion set schema_version = '1.1.65',  updated= CURRENT_TIMESTAMP();
+
+-- Version 1.1.66
+
Index: /branches/eam_branches/ipp-20100823/dbconfig/config.md
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/config.md	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/config.md	(revision 29515)
@@ -2,4 +2,4 @@
     pkg_name        STR     ippdb
     pkg_namespace   STR     ippdb
-    pkg_version     STR     1.1.63
+    pkg_version     STR     1.1.65
 END
Index: /branches/eam_branches/ipp-20100823/dbconfig/magic.md
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/magic.md	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/magic.md	(revision 29515)
@@ -42,4 +42,5 @@
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     uri         STR         255
+    path_base   STR         255
     streaks     S32         0
     fault       S16         0       # Key
Index: /branches/eam_branches/ipp-20100823/dbconfig/pstamp.md
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/pstamp.md	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/pstamp.md	(revision 29515)
@@ -46,4 +46,5 @@
     options     S64         64
     dep_id      S64         0
+    fault_count S32         0
 END
 
@@ -59,4 +60,5 @@
     outdir      STR         255
     fault       S16         0
+    fault_count S32         0
 END
 
Index: /branches/eam_branches/ipp-20100823/dbconfig/stack.md
===================================================================
--- /branches/eam_branches/ipp-20100823/dbconfig/stack.md	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/dbconfig/stack.md	(revision 29515)
@@ -49,4 +49,5 @@
     hostname           STR    64
     good_frac          F32    0.0     # Key
+    mjd_obs            F64    0.0
     fault              S16    0       # Key
     software_ver    STR         16
Index: /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in	(revision 29515)
@@ -16,4 +16,5 @@
 $(DESTBIN)/czartool_getServerStatus.pl \
 $(DESTBIN)/czartool_revert.pl \
+$(DESTBIN)/czartool_serverstoprun.pl \
 $(DESTBIN)/build_histogram.dvo \
 $(DESTBIN)/helpers.dvo \
@@ -205,5 +206,8 @@
 $(DESTWWW)/scatterCpiBgReMaImage.php \
 $(DESTWWW)/scatterCpiBgReMpImage.php \
-$(DESTWWW)/scatterCpiBgReSaImage.php
+$(DESTWWW)/scatterCpiBgReSaImage.php \
+$(DESTWWW)/exposureStatus.php \
+$(DESTWWW)/mopsStatus.php \
+$(DESTWWW)/gpc1MysqlProcessList.php
 
 PICTURES = \
Index: /branches/eam_branches/ipp-20100823/ippMonitor/def/autocode.php
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/def/autocode.php	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/def/autocode.php	(revision 29515)
@@ -25,5 +25,5 @@
 $restricted = 0;
 
-// define restrictiosn to the queries
+// define restrictions to the queries
 // ** TABLE RESTRICTIONS **
 
@@ -33,4 +33,7 @@
   }
 }
+
+// Add GROUP BY statements if any
+// ** GROUP RESTRICTIONS **
 
 // get the result table count
Index: /branches/eam_branches/ipp-20100823/ippMonitor/def/exposureStatus.d
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/def/exposureStatus.d	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/def/exposureStatus.d	(revision 29515)
@@ -0,0 +1,32 @@
+TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN distRun ON stage = 'camera' AND distRun.stage_id = cam_id 
+TITLE Exposures Status
+FILE exposureStatus.php
+MENU ipp.imfiles.dat
+
+WHERE object != 'ENGINEERING'
+
+UNRESTRICTED AND dateobs >= CURDATE()
+
+#     field                    width format  name          show         link to                   extras
+FIELD DISTINCT rawExp.exp_id, 1,    %s,    'Dont use for filtering'
+FIELD dateobs, 15,    %T,     Date obs
+FIELD rawExp.exp_id, 5, %d,      Exp ID
+FIELD exp_name, 5,    %s,     Exp Name
+FIELD chipRun.data_group, 5,   %s,     Data grp
+FIELD chip_id, 5,    %d,     Chip ID
+FIELD chipRun.state, 5,    %s,     Chip State
+FIELD cam_id, 5,    %d,     Cam ID
+FIELD camRun.state, 5,    %s,     Cam ID
+FIELD camRun.magicked, 5,    %s,     Cam Magicked
+#FIELD publishRun.pub_id, 5,    %d,     Pub ID
+FIELD distRun.dist_id, 5,    %d,     Dist ID
+FIELD fake_id, 5,    %d,     Fake ID
+FIELD warp_id, 5,    %d,     Warp ID
+FIELD warpRun.state, 5,    %s,     Warp State
+FIELD diffInput.diff_id, 5,    %d,     Diff ID
+FIELD diffInputRun.state, 5,    %s,     Diff State
+FIELD diffTemplate.diff_id, 5,    %d,     Diff Templ ID
+FIELD diffTemplateRun.state, 5,    %s,     Diff Templ State
+FIELD filter, 5,    %s,     Filter
+FIELD object, 25,    %s,     Object
+FIELD comment, 50,    %s,     Comment
Index: /branches/eam_branches/ipp-20100823/ippMonitor/def/gpc1MysqlProcessList.d
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/def/gpc1MysqlProcessList.d	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/def/gpc1MysqlProcessList.d	(revision 29515)
@@ -0,0 +1,13 @@
+TABLE INFORMATION_SCHEMA.PROCESSLIST
+TITLE GPC 1 MySql Process List
+FILE gpc1MysqlProcessList.php
+MENU ipp.imfiles.dat
+
+FIELD ID,   5,  %d, Process ID
+FIELD USER, 10, %s, User
+FIELD HOST, 30, %s, Host:port
+FIELD DB,   10, %s, Database
+FIELD TIME, 10, %s, Time
+FIELD STATE, 10, %s, State
+FIELD INFO, 40, %s, Info
+FIELD TIME_MS, 10, %s, Time (ms)
Index: /branches/eam_branches/ipp-20100823/ippMonitor/def/mopsStatus.d
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/def/mopsStatus.d	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/def/mopsStatus.d	(revision 29515)
@@ -0,0 +1,38 @@
+TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id
+TITLE MOPS Status
+FILE mopsStatus.php
+MENU ipp.imfiles.dat
+
+UNRESTRICTED WHERE 0=1
+
+# OP  OP1   ($ddiff_id > 0 ? $ddiff_id : $tdiff_id)
+# OP  OP1   $ddiffId.":".$tdiffId
+OP OP1 ($row[8]>$row[9]?$row[8]. "(1)":($row[9]!=0?$row[9]. "(2)":"0"))
+# $diffInput.diff_id+$diffTemplate.diff_id
+OP OP2 $row[8]+$row[9]
+#  "[".$diffInput.diff_id."]:[".$tdiff_id."]"
+
+#     field                    width format  name          show         link to                   extras
+FIELD dateobs, 		       15,   %T,     Date Obs
+FIELD exp_name, 	       5,    %s,     Exp Name
+FIELD rawExp.exp_id, 	       5,    %d,     Exp ID
+FIELD chipRun.data_group,      5,    %s,     Data grp
+FIELD chip_id, 		       5,    %d,     Chip ID (Stage 1)
+FIELD cam_id, 		       5,    %d,     Cam ID (Stage 2)
+FIELD fake_id, 		       5,    %d,     Fake ID (Stage 3)
+FIELD warp_id, 		       5,    %d,     Warp ID (Stage 4)
+FIELD diffInput.diff_id AS ddiff_id,       5,    %d,     ddiff_id, none
+FIELD diffTemplate.diff_id AS tdiff_id,    5,    %d,     tdiff_id, none
+FIELD *,		       5,    %s,     Diff ID (Stage 5),		op=OP1
+#FIELD *,		       5,    %d,     Diff ID (Stage 5),		op=OP2
+FIELD camRun.magicked, 	       5,    %d,     Cam Magicked (Stage 6)
+FIELD publishRun.pub_id,       5,    %d,     Pub ID (Stage 7)
+FIELD filter, 		       5,    %s,     Filter
+FIELD object, 		       15,    %s,     Object
+FIELD comment, 		       40,    %s,     Comment
+
+FIELD diffInput.diff_id AS ddiffId,       5,    %s,     ddiffId,	none
+FIELD diffTemplate.diff_id AS tdiffId,    5,    %s,     tdiffId, 	none
+
+MODE summary
+GROUP rawExp.exp_id
Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_getplot.php
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_getplot.php	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_getplot.php	(revision 29515)
@@ -15,9 +15,10 @@
 $label = $_GET[label];
 $stage = $_GET[stage];
+$plottype = $_GET[plottype];
 
 if ($type=="t")
-$filePath = "/tmp/czarplot_".$label."_".$stage."_".$type.".png";
+$filePath = "/tmp/czarplot_".$plottype."_".$label."_".$stage."_".$type.".png";
 else if ($type=="h")
-$filePath = "/tmp/czarplot_".$label."_all_stages_".$type.".png";
+$filePath = "/tmp/czarplot_".$plottype."_".$label."_all_stages_".$type.".png";
 else if ($type=="s")
 $filePath = "/tmp/czarplot_hosts_space.png";
@@ -29,4 +30,5 @@
 }
 
+
 exit();
 
Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php	(revision 29515)
@@ -25,45 +25,4 @@
 
 menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
-
-$pass = $ID['pass'];
-$proj = $ID['proj'];
-$menu = $ID['menu'];
-
-$selectedStage = $_GET[stage];
-$selectedLabel = $_GET[label];
-$selectedServer = $_GET[server];
-$selectedRevertStage = $_GET[revertstage];
-$selectedRevertMode = $_GET[revertmode];
-
-if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
-if ($selectedStage == "") { $selectedStage = "all_stages"; }
-
-$nsStatus = getNightlyScienceStatus($czardb);
-echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus</p>";
-
-
-// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
-if ($selectedRevertStage != "" && $selectedRevertMode != "") {
-
-    exec("czartool_revert.pl -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
-    $currentRevertMode;
-    if ($response[0] == "off") $currentRevertMode = 0;
-    else if ($response[0] == "on") $currentRevertMode = 1;
-
-    setRevertStatus($czardb, $selectedRevertStage, $currentRevertMode);
-}
-$debug = 0;
-
-$stdsLabels = getLabels($czardb, "stdscience");
-$distLabels = getLabels($czardb, "distribution");
-$pubLabels = getLabels($czardb, "publishing");
-$updateLabels = getLabels($czardb, "update");
-
-if ($debug) {
-    echo "prog returned $status, and output:<br>";
-    for ($i = 0; $i < count($distLabels); $i++) {
-        echo "$distLabels[$i]<br>";
-    }
-}
 
 $states=array("full","new","drop","wait");
@@ -82,4 +41,55 @@
         "summitcopy");
 
+$pass = $ID['pass'];
+$proj = $ID['proj'];
+$menu = $ID['menu'];
+
+$selectedStage = $_GET[stage];
+$selectedLabel = $_GET[label];
+$selectedServer = $_GET[server];
+$selectedRevertStage = $_GET[revertstage];
+$selectedRevertMode = $_GET[revertmode];
+$plotType = $_GET[plottype];
+$serverCmd = $_GET[servercmd];
+$allServerCmd = $_GET[allservercmd];
+
+if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
+if ($selectedStage == "") { $selectedStage = "all_stages"; }
+if ($plotType == "") { $plotType = "linear"; }
+
+$nsStatus = getNightlyScienceStatus($czardb);
+$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
+$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
+echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots </p>";
+
+
+// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
+if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode);
+
+// tell selected server to stop or run
+if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer,  $serverCmd);
+
+// tell all servers to stop or run
+if ($allServerCmd) {
+
+    foreach ($servers as &$server) serverStopRun($czardb, $server,  $allServerCmd);
+}
+
+
+$debug = 0;
+
+$stdsLabels = getLabels($czardb, "stdscience");
+$distLabels = getLabels($czardb, "distribution");
+$pubLabels = getLabels($czardb, "publishing");
+$updateLabels = getLabels($czardb, "update");
+
+if ($debug) {
+    echo "prog returned $status, and output:<br>";
+    for ($i = 0; $i < count($distLabels); $i++) {
+        echo "$distLabels[$i]<br>";
+    }
+}
+
+
 // set up the form
 echo "<form action=\"czartool_labels.php\" method=\"POST\">\n";
@@ -91,51 +101,51 @@
 
 echo "<table>\n";
-  echo "<tr>\n";
-    echo "<td>\n";
-      echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage\"><br>";
-    echo "</td>\n";
-
-    echo "<td> \n";
-      createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
-    echo "</td>\n";
-  echo "</tr>\n";
-
-  echo "<tr>\n";
-    echo "<td>\n";
-      echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage\"><br>";
-    echo "</td>\n";
-
-    echo "<td> \n";
-      createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
-    echo "</td>\n";
-  echo "</tr>\n";
-
-  echo "<tr>\n";
-    echo "<table>\n";
-      echo "<tr valign=top>\n";
-        echo "<td> \n";
-          echo "<img src=\"czartool_getplot.php?type=s\"><br>";
-        echo "</td>\n";
-        echo "<td>\n";
-          createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage);
-        echo "</td>\n";
-
-        echo "<td> \n";
-          $today = date("Y-m-d");
-          showSummitData($gpc1db, $today);
-        echo "</td>\n";
-      echo "</tr>\n";
-    echo "</table>\n";
-
-    echo "<table>\n";
-      echo "<tr valign=top>\n";
-        echo "<td> \n";
-        echo "</td>\n";
-        echo "<td>\n";
-      if ($selectedServer) showServerStatus($selectedServer);
-        echo "</td>\n";
-      echo "</tr>\n";
-    echo "</table>\n";
-  echo "</tr>\n";
+echo "<tr>\n";
+echo "<td>\n";
+echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage&plottype=$plotType\"><br>";
+echo "</td>\n";
+
+echo "<td> \n";
+createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
+echo "</td>\n";
+echo "</tr>\n";
+
+echo "<tr>\n";
+echo "<td>\n";
+echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage&plottype=linear\"><br>";
+echo "</td>\n";
+
+echo "<td> \n";
+createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
+echo "</td>\n";
+echo "</tr>\n";
+
+echo "<tr>\n";
+echo "<table>\n";
+echo "<tr valign=top>\n";
+echo "<td> \n";
+echo "<img src=\"czartool_getplot.php?type=s\"><br>";
+echo "</td>\n";
+echo "<td>\n";
+createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage, $plotType);
+echo "</td>\n";
+
+echo "<td> \n";
+$today = date("Y-m-d");
+showSummitData($gpc1db, $today);
+echo "</td>\n";
+echo "</tr>\n";
+echo "</table>\n";
+
+echo "<table>\n";
+echo "<tr valign=top>\n";
+echo "<td> \n";
+echo "</td>\n";
+echo "<td>\n";
+if ($selectedServer && !$serverCmd) showServerStatus($selectedServer);
+echo "</td>\n";
+echo "</tr>\n";
+echo "</table>\n";
+echo "</tr>\n";
 echo "</table>\n";
 
@@ -225,5 +235,16 @@
 #
 ###########################################################################
-function createLabelsTable($pass, $proj, $db, $server, $labels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage) {
+function createLabelsTable(
+        $pass, 
+        $proj, 
+        $db, 
+        $server, 
+        $labels, 
+        $distLabels, 
+        $pubLabels, 
+        $stages, 
+        $states, 
+        $selectedState, 
+        $selectedLabel, $selectedStage, $plotType) {
 
     // set up table columns
@@ -241,5 +262,13 @@
         if ($stage == "burntool") continue;
         $reverting = getRevertStatus($db, $stage);
-        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode=";
+        $link = 
+            "czartool_labels.php?pass=".$pass
+            ."&proj=".$proj
+            ."&label=".$selectedLabel
+            ."&stage=".$selectedStage
+            ."&plottype=".$plotType
+            ."&revertstage=".$stage
+            ."&revertmode=";
+
         if(!$reverting) {$label =  "Start";$link = $link . "on";}
         if($reverting) {$label = "Stop";$link = $link . "off";}
@@ -254,12 +283,24 @@
     write_header_cell($class, "Label (in order of priority)");
     foreach ($stages as &$stage) {
-        
+
         if ($stage == $selectedStage) $link = "";
-        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $stage;
+        else $link = 
+            "czartool_labels.php?pass=".$pass
+                ."&proj=".$proj 
+                ."&label=".$selectedLabel
+                ."&stage=".$stage
+                ."&plottype=".$plotType;
+
         write_table_cell($class, '%s', $link, $stage);
     }
 
     if ($selectedStage=="all_stages") $link = "";
-    else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=all_stages";
+    else $link = 
+        "czartool_labels.php?pass=".$pass
+            ."&proj=".$proj
+            ."&label=".$selectedLabel
+            ."&stage=all_stages"
+            ."&plottype=".$plotType;
+
     write_table_cell($class, '%s', $link, "All stages");
 
@@ -267,5 +308,10 @@
     echo "<tr><td></td>\n";
 
-    $defaultlink = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj;
+    $defaultlink = 
+        "czartool_labels.php?pass=".$pass
+        ."&proj=".$proj
+        ."&label=".$selectedLabel
+        ."&stage=".$selectedStage
+        ."&plottype=".$plotType;
 
     // write rows
@@ -285,5 +331,10 @@
         // create link to label summary page for each label
         if ($thisLabel == $selectedLabel) $link = "";
-        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $thisLabel . "&stage=" . $selectedStage;
+        else $link = 
+            "czartool_labels.php?pass=".$pass
+                ."&proj=".$proj
+                ."&label=".$thisLabel
+                ."&stage=".$selectedStage
+                ."&plottype=".$plotType;
 
         echo "<tr><td></td>\n";
@@ -342,5 +393,5 @@
 
     if ($selectedLabel == "all_".$server."_labels") $link = "";
-    else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage;
+    else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage."&plottype=".$plotType;
 
     echo "<tr><td></td>\n";
@@ -385,4 +436,19 @@
 ###########################################################################
 #
+#  Turns reverts on or off for a given task
+#
+###########################################################################
+function turnRevertsOnOff($db, $stage, $mode) {
+
+    exec("czartool_revert.pl -t $stage -o $mode", $response, $status);
+    $currentRevertMode;
+    if ($response[0] == "off") $currentRevertMode = 0;
+    else if ($response[0] == "on") $currentRevertMode = 1;
+
+    setRevertStatus($db, $stage, $currentRevertMode);
+}
+
+###########################################################################
+#
 # Returns whether this stage is reverting or not
 #
@@ -420,5 +486,8 @@
     $faults = $row[1];
 
-    $str = "$pending";
+    if ($pending == 0)
+        $str = "";
+    else
+        $str = "$pending";
 
     if ($state == "new") {
@@ -437,5 +506,5 @@
 #
 ###########################################################################
-function createServersTable($pass, $proj, $db, $servers, $selectedLabel, $selectedStage) {
+function createServersTable($pass, $proj, $db, $servers, $selectedLabel, $selectedStage, $plotType) {
 
     // set up table columns
@@ -445,18 +514,65 @@
     write_header_cell($class, "Server");
     write_header_cell($class, "Alive?");
-    write_header_cell($class, "Scheduler running?");
+    $link = "czartool_labels.php?pass=".$pass
+        ."&proj=".$proj
+        ."&label=".$selectedLabel
+        ."&stage=".$selectedStage
+        ."&plottype=".$plotType
+        ."&allservercmd=stop";
+
+    write_table_cell($class, '%s', $link, "Stop all");
+    $link = "czartool_labels.php?pass=".$pass
+        ."&proj=".$proj
+        ."&label=".$selectedLabel
+        ."&stage=".$selectedStage
+        ."&plottype=".$plotType
+        ."&allservercmd=run";
+
+    write_table_cell($class, '%s', $link, "Run all");
     echo "</tr>\n";
 
     foreach ($servers as &$server) {
 
-        $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&server=".$server."&label=".$selectedLabel."&stage=".$selectedStage;
-        //      $link = "";
-
         getServerStatus($db, $server, $alive, $running);
 
         echo "<tr><td></td>\n";
+        $link = "czartool_labels.php?pass=".$pass
+            ."&proj=".$proj
+            ."&server=".$server
+            ."&label=".$selectedLabel
+            ."&stage=".$selectedStage
+            ."&plottype=".$plotType;
+
         write_table_cell($class, '%s', $link, $server);
         write_table_cell($class, '%s', "", $alive ? "yes" : "NO");
-        write_table_cell($class, '%s', "", $running ? "yes" : "NO");
+
+        if ($alive) {
+
+            $link = "czartool_labels.php?pass=".$pass
+                ."&proj=".$proj
+                ."&label=".$selectedLabel
+                ."&stage=".$selectedStage
+                ."&plottype=".$plotType
+                ."&server=".$server
+                ."&servercmd=";
+
+            if ($running)  {
+
+                $link = $link . "stop";
+                write_table_cell($class, '%s', $link, "stop");
+                write_table_cell($class, '%s', "", "");
+            }
+            else {
+
+                $link = $link . "run";
+                write_table_cell($class, '%s', "", "");
+                write_table_cell($class, '%s', $link, "run");
+            }
+        }
+        else {
+
+            write_table_cell($class, '%s', "", "");
+            write_table_cell($class, '%s', "", "");
+        }
         echo "</tr>\n";
     }
@@ -464,4 +580,36 @@
     echo "</table>\n";
 }
+
+###########################################################################
+#
+#  Commands a server to stop or run 
+#
+###########################################################################
+function serverStopRun($db, $server, $cmd) {
+
+    exec("czartool_serverstoprun.pl -s $server -c $cmd", $response, $status);
+    $alive = 0;
+    $running = 0;
+    if ($response[0] == "running") {$alive = 1; $running = 1;}
+    else if ($response[0] == "stopped") {$alive = 1; $running = 0;} 
+    else if ($response[0] == "dead") {$alive = 0; $running = 0;}
+    setServerStatus($db, $server, $alive, $running);
+}
+
+###########################################################################
+#
+# Sets server status 
+#
+###########################################################################
+function setServerStatus($db, $server, $alive, $running) {
+
+    $sql = "INSERT INTO servers (server, alive, running) VALUES ('$server', $alive, $running)";
+
+    if ($debug) {echo "$sql<br>";}
+
+    $qry = $db->query($sql);
+    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
+}
+
 
 ###########################################################################
Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat	(revision 29515)
@@ -39,2 +39,7 @@
 menulink  | menuselect 	 | link    | Clean /tmp directory         | cleanTmpDirectory.php
 menutop   | menutop      | link    | disk usage                   | diskUsage.php
+
+menutop   | menutop      | plain   | &nbsp;                       | 
+menulink  | menuselect 	 | link    | Exposures Status             | exposureStatus.php
+menulink  | menuselect 	 | link    | MOPS Exposures Status        | mopsStatus.php
+menulink  | menuselect 	 | link    | GPC1 MySql ProcessList            | gpc1MysqlProcessList.php
Index: /branches/eam_branches/ipp-20100823/ippMonitor/scripts/czartool_serverstoprun.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/scripts/czartool_serverstoprun.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/scripts/czartool_serverstoprun.pl	(revision 29515)
@@ -0,0 +1,29 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long;
+
+my $server = undef;
+my $cmd = undef;
+
+GetOptions (
+        "server|s=s" => \$server,
+        "cmd|c=s" => \$cmd);
+
+if (!$server) {print "ERROR: need to define a server (-s)\n"; exit;}
+if (!$cmd) {print "ERROR: need to define a cmd (-c)\n"; exit;}
+if (($cmd !~ m/^stop$/) && ($cmd !~ m/^run$/)) {print "ERROR: cmd must be 'stop' or 'run'\n"; exit;}
+
+my @cmdOut = `echo "$cmd; status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+
+my $line;
+foreach $line (@cmdOut) {
+
+    chomp($line);
+    if ($line =~ m/Scheduler is stopped/) {print "stopped\n"; exit;}
+    if ($line =~ m/Scheduler is running/) {print "running\n"; exit;}
+    if ($line =~ m/Task Status/) {last;}
+}
+
+print "dead\n";
Index: /branches/eam_branches/ipp-20100823/ippMonitor/scripts/generate
===================================================================
--- /branches/eam_branches/ipp-20100823/ippMonitor/scripts/generate	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippMonitor/scripts/generate	(revision 29515)
@@ -227,4 +227,9 @@
         }
 
+        # fill in GROUP BY
+        if ($line =~ m|// \*\* GROUP RESTRICTIONS \*\*|) {
+            &write_group_by;
+        }
+
         # fill in table restricts
         if ($line =~ m|// \*\* BUTTON RESTRICTIONS \*\*|) {
@@ -331,4 +336,9 @@
         print FILE "\$WHERE = check_restrict ('$value', \$WHERE, 'max', 1.0);\n";
     }
+    # print FILE "\$WHERE = check_ordering ('$GROUPS', \$WHERE);\n";
+}
+
+# Generate GROUP BY statement if any 
+sub write_group_by {
     print FILE "\$WHERE = check_ordering ('$GROUPS', \$WHERE);\n";
 }
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl	(revision 29515)
@@ -124,9 +124,10 @@
     defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode or
-    defined $confirm_stacks;
+    defined $confirm_stacks or defined $burntool_stats;
 
 # Configurable parameters from our config file.
 my @target_list = ();
 my @filter_list = ();
+my %distribution_list = ();
 my %tessID_list = ();
 my %obsmode_list = ();
@@ -135,4 +136,5 @@
 my %cleanmods_list = ();
 my %stackable_list = ();
+my %extra_processing = ();
 my %reduction_class = ();
 my %macro_formats = ();
@@ -206,4 +208,8 @@
                 $this_target = ${ $tentry }{value};
                 push @target_list, $this_target;
+		$distribution_list{$this_target} = $this_target;
+            }
+            elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
+                $distribution_list{$this_target} = ${ $tentry }{value};
             }
             elsif (${ $tentry }{name} eq 'TESS') {
@@ -222,4 +228,7 @@
                 $stackable_list{$this_target} = ${ $tentry }{value};
             }
+	    elsif (${ $tentry }{name} eq 'EXTRA_PROCESSING') {
+		$extra_processing{$this_target} = ${ $tentry }{value};
+	    }
 	    elsif (${ $tentry }{name} eq 'REDUCTION') {
 		$reduction_class{$this_target} = ${ $tentry }{value};
@@ -726,14 +735,14 @@
         my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target);
 	if (defined($burntool_stats)) {
-	    print "$Nexposures $Nimfiles $Nburntooled $Nalready\n";
+	    print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n";
 	}
 
         if ($Nexposures == 0) {
 	    print STDERR "execute_chips: Target $target on $date had no exposures.\n";
-#	    next;
+	    next;
         }
         if ($Nalready != 0) {
 	    print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
-#            next;
+            next;
         }
         if ($Nimfiles != $Nburntooled) {
@@ -1183,4 +1192,6 @@
         &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     }
+    
+    
     return(0);
 }
@@ -1190,6 +1201,9 @@
     my $pretend = shift;
     foreach my $target (@target_list) {
-        if ($stackable_list{$target} == 1) {
-            foreach my $filter (@filter_list) {
+	foreach my $filter (@filter_list) {
+	    if (exists($extra_processing{$target})) {
+		do_extra_processing($date,$target,$filter,$pretend);
+	    }
+	    if ($stackable_list{$target} == 1) {
                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
@@ -1231,10 +1245,9 @@
 		}
             }
+	    else {
+		# print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+	    }
         }
-        else {
-            # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
-        }
-    }
-
+    }
 }
 
@@ -1291,4 +1304,71 @@
 
 #
+# Extra processing
+################################################################################
+
+sub do_extra_processing {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    if ($target eq 'OSS') {
+	my $db = init_gpc_db();
+
+	my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+	$obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+	print STDERR "$obj_sth\n";
+	my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+	foreach my $object_row (@{ $object_ref }) {
+	    my $this_object = shift @{ $object_row };
+	    my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+	    $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+	    $input_sth .= " ORDER BY dateobs ";
+
+	    my $warps = $db->selectall_arrayref( $input_sth );
+	    
+	    if (($#{ $warps } + 1) % 2 != 0) {
+		print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
+		die;
+	    }
+	    
+	    while ($#{ $warps } > -1) {
+		my $input_warp = shift @{ $warps };
+		my $template_warp = shift @{ $warps };
+		my $input_exp_id = ${ $input_warp }[0];
+		my $template_exp_id = ${ $template_warp }[0];
+		
+		my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+		$cmd .= "-input_label $label  -template_label $label ";
+		$cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+		$cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+		$cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#		$cmd .= " -pretend ";
+		if (defined($pretend)) {
+		    $cmd .= ' -pretend ';
+		}
+		if ($debug == 1) {
+		    $cmd .= ' -pretend ';
+		}
+		print STDERR "EXTRA_PROCESSING: $cmd\n";
+		if (($debug == 0)&&(!defined($pretend))) {
+		    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+			run ( command => $cmd, verbose => $verbose );
+		    unless ($success) {
+			$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+			&my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		    }
+		}
+	    }
+	}
+    }
+}
+
+	    
+
+
+#
 # Auto-Clean
 ################################################################################
@@ -1380,5 +1460,5 @@
     my $object   = $object_list{$target};
     my $comment = $comment_list{$target};
-    my $dist_group = $target;
+    my $dist_group = $distribution_list{$target};
     my $data_group = "${target}.${trunc_date}";
     my $tess_id = $tessID_list{$target};
@@ -1427,5 +1507,5 @@
 	my $N = $metadata_out{N_MACROS};
 	$metadata_out{"ns${N}Macro"} = $macro_formats{$proc_mode};
-	print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
+#	print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
 	if (defined($date)&&(defined($target))) {
 
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl	(revision 29515)
@@ -55,4 +55,5 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1);
+my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1);
 #my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
 if ($missing_tools) {
@@ -214,4 +215,5 @@
 
         $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
+	$status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
         $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
         if ($status) {
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_cleanup.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/ipp_cleanup.pl	(revision 29515)
@@ -53,8 +53,14 @@
 }
 
+# set this to 1 to enable checking for files on dead nodes
+# it is off for now because the implementation is a hack
+# See comments below.
+my $check_for_gone = 0;
+
 my $error_state;
-if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
-if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
-if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
+my $done_state;
+if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned"; $done_state = "cleaned"; }
+if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; $done_state = "scrubbed";}
+if ($mode eq "goto_purged")   { $error_state = "error_purged";   $done_state = "purged";}
 
 
@@ -92,8 +98,9 @@
     }
 
-    # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
-    # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run.
+    # if there are no chipProcessedImfiles (@$stdout_buf == 0) then assume that we're done
+    # it could be that there are no chipProcessedImfiles at all if say if a run was changed from drop to goto_cleaned
+    # or of a run was set to update and then back to goto_cleaned before any images were processed
     if (@$stdout_buf == 0)  {
-        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state";
+        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -126,7 +133,12 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for chipRun $stage_id $class_id "
-                        . " because config file ($config_file) is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -354,7 +366,9 @@
 
     if (@$stdout_buf == 0) {
-        # No skycells were found for some reason.
-        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
-        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state";
+        # No skycells were found for some reason. 
+        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
+        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
+        # updated
+        my $command = "$warptool -updaterun -warp_id $stage_id -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -384,7 +398,12 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
-                        " because config file is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
+                            " because config file is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
+                            " because config file is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -624,7 +643,8 @@
 
     if (@$stdout_buf == 0) {
-        # No skycells were found for some reason.
-        # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself."
-        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
+        # No skycells were found for some reason. 
+        # it could be that there are no warpSkyfiles at all if say if a run was changed from drop to goto_cleaned
+        # or of a run was cleaned, set to update, and then back to goto_cleaned before any images were successfully
+        my $command = "$difftool -updaterun -diff_id $stage_id -set_state $done_state";
         $command .= " -dbname $dbname" if defined $dbname;
 
@@ -658,7 +678,12 @@
 
                 unless ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
-                        " because config file ($config_file) is missing\n";
-                    $status = 0;
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for diffRun $stage_id $skycell_id" .
+                            " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
+                            " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
                 }
             }
@@ -1665,4 +1690,59 @@
 }
 
+my $whichnode;
+sub file_gone
+{
+    # if $check_for_gone check whether the only instance of file is on a lost volumen
+    # XXX: we don't have a proper interface for this. 
+    # For now try to use Bill's hack the script 'whichnode'
+    return 0 if !$check_for_gone;
+
+    my $file = shift;
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode", "chip", $stage_id, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", "chip", $stage_id, $error_code);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone node return 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
 sub addFilename
 {
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak.pl	(revision 29515)
@@ -42,4 +42,5 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
+my ($streaks_path_base, $inv_streaks_path_base);
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
@@ -49,4 +50,7 @@
            'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
+           'streaks_path_base=s'      => \$streaks_path_base,    # path_base for streaks data
+           'inv_streaks_path_base=s'  => \$inv_streaks_path_base, #path_base for streaks from inverse diff
+           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
@@ -78,4 +82,5 @@
     defined $camera and
     defined $streaks and
+    defined $streaks_path_base and
     defined $stage and
     defined $stage_id and
@@ -108,4 +113,5 @@
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
+$inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL");
 $inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
 
@@ -240,4 +246,20 @@
     my ($allstreaks_fh, $allstreaks_name);
 
+    # Set name of streaks files from their path_base. Note. ne 'NULL' test is for backward compatability
+    # with runs that don't have path_base set yet
+    if ($streaks_path_base ne 'NULL') {
+        $streaks = "$streaks_path_base.streaks";
+    }
+    if ($inv_streaks_path_base and ($inv_streaks_path_base ne 'NULL')) {
+        $inv_streaks = "$inv_streaks_path_base.streaks";
+    }
+
+    my $streaks_resolved = $ipprc->file_resolve($streaks) or &my_die("failed to resolve streaks file $streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    my $inv_streaks_resolved;
+    if ($inv_streaks) {
+        $inv_streaks_resolved = $ipprc->file_resolve($inv_streaks) or &my_die("failed to resolve inverse streaks file $inv_streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    }
+        
+
     if ($stage eq "raw") {
         $image = $uri;
@@ -301,5 +323,5 @@
         $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
 
-        if ($inv_streaks) {
+        if ($inv_streaks_resolved) {
             # create a temporary file containing the contents of the
             # two streaks files
@@ -307,13 +329,13 @@
                     UNLINK => !$save_temps);
 
-            combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
+            combine_streaks($allstreaks_fh, $streaks_resolved, $inv_streaks_resolved);
 
             # apply the combined streaks to both the forward and inverse diffs
-            $streaks = $allstreaks_name;
+            $streaks_resolved = $allstreaks_name;
         }
     }
 
     {
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
 
         $command .= " -stats $statsFile";
@@ -340,5 +362,5 @@
         }
     }
-    if (($stage eq "diff") and $inv_streaks) {
+    if (($stage eq "diff") and $inv_streaks_resolved) {
         $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
         $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
@@ -349,5 +371,5 @@
         my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
 
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
         $command .= " -stats $invStatsFile";
 
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl	(revision 29515)
@@ -17,4 +17,5 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Copy;
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -41,5 +42,5 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
 
 GetOptions(
@@ -49,4 +50,5 @@
            'dbname=s'        => \$dbname,     # Database name
            'baseroot=s'      => \$baseroot,   # Output root name
+           'final-outroot=s' => \$final_outroot,   # location for final outputs
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -86,4 +88,16 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# list of VerifyStreaks input and output files to copy to nebulous 
+my @verify_outputs = qw(
+clusterPos.txt
+duplicate.png
+mask.png
+original.png
+original.fits
+residual.png
+residual.fits
+clusters.list
+);
 
 ### Get a list of inputs
@@ -311,5 +325,4 @@
 }
 
-
 ### Input result into database
 {
@@ -334,4 +347,7 @@
 
 if ($node eq "root") {
+    # XXXX: Since we just added the result above, all of these my_dies are going to fail
+    # with a duplicate row error.
+    # see more comments below
     my $streaks_file = "$outroot.streaks";
     my $resolved = $ipprc->file_resolve($streaks_file);
@@ -349,5 +365,4 @@
     }
 
-    &run_verifystreaks($baseroot);
 
     my $exp_id;                 # Exposure identifier
@@ -366,4 +381,6 @@
     }
 
+    &run_verifystreaks($baseroot, $exp_id);
+
     {
         my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
@@ -383,9 +400,20 @@
     }
 
+    my $output_streaks = $final_outroot . ".streaks";
+    if ($output_streaks and ($output_streaks ne $streaks_file)) {
+        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
+        foreach my $f (@verify_outputs) {
+            my $src = "$baseroot.verify/${exp_id}_$f";
+            my $dest = "$final_outroot.${f}";
+            copy_to_nebulous($ipprc, $src, $dest, 1);
+        }
+    } else {
+        $output_streaks = $streaks_file;
+    }
 
     {
         my $command = "$magictool -addmask";
         $command   .= " -magic_id $magic_id";
-        $command   .= " -uri $streaks_file";
+        $command   .= " -path_base $final_outroot";
         $command   .= " -streaks $num_streaks";
         $command   .= " -dbname $dbname" if defined $dbname;
@@ -397,5 +425,9 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                # This isn't going to work because our result was already entered.
+                # XXX: This my_die isn't going to work because the result for the root node was already
+                # added to the database up above.
+                # There is a magicMask has a fault column. Maybe we should 
+                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
+                # for the root node.
                 &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
             }
@@ -406,4 +438,5 @@
 }
 
+
 ### Pau.
 
@@ -411,4 +444,5 @@
 
     my $baseroot = shift;
+    my $exp_id = shift;
 
     unless ($VerifyStreaks) {
@@ -431,6 +465,7 @@
     my @files = <$baseroot.*.clusters>;
 
-    unless (open ($FILE, ">$outdir/clusters.list")) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+    my $clusters_list = "$outdir/${exp_id}_clusters.list";
+    unless (open ($FILE, ">$clusters_list")) {
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
@@ -441,9 +476,9 @@
     close ($FILE);
     if ($status) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
 
-    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -513,4 +548,40 @@
 }
 
+# Copy a file to nebulous and optionally replicate it
+# We should consider making this an ipprc function. For now try it here so we can print
+# the right error messages
+sub copy_to_nebulous {
+    my $ipprc = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $replicate = shift;
+
+    print "copying $src to $dest\n";
+
+    $ipprc->file_exists($src) or
+        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    # copy the file to it's final destination - which is presumably in nebulous
+    # we delete it so that all instances are deleted in case it has been replicated
+    if ($ipprc->file_exists($dest)) {
+        $ipprc->file_delete($dest) or 
+                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
+    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
+
+    copy ($src, $dest_resolved) or 
+        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($replicate and (file_scheme($dest) eq 'neb')) {
+        my $neb = $ipprc->nebulous();
+        $neb->setxattr($dest, 'user.copies', 2, 'create') or 
+            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+        $neb->replicate($dest) or
+            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
 sub my_die
 {
Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_tree.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_tree.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_tree.pl	(revision 29515)
@@ -83,8 +83,14 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to perform magictool -inputskyfile: $error_code", $magic_id, $error_code);
+    }
+
+    my $magictool_output = join "", @$stdout_buf;
+    # if there is no output from magictool that means that there are no
+    # diffSkyfiles with non-zero quality. Set fault to a special value so
+    # that these magicRuns can be recognized and dropped.
+    &my_die("magictool -inputskyfile returned no output. Inputs are probably all bad quality.", $magic_id, 42)
+        if !$magictool_output;
+    my $metadata = $mdcParser->parse($magictool_output) or
         &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
 
Index: /branches/eam_branches/ipp-20100823/ippTasks/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/Makefile.am	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/Makefile.am	(revision 29515)
@@ -18,4 +18,5 @@
 	magic.pro \
 	destreak.pro \
+	destreak.cleanup.pro \
 	diff.pro \
 	stack.pro \
Index: /branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTasks/destreak.cleanup.pro	(revision 29515)
@@ -0,0 +1,172 @@
+## destreak.cleanup.pro : tasks for the cleaning up destreak stage : -*- sh -*-
+
+# test for required global variables
+check.globals
+
+$LOGSUBDIR = $LOGDIR/destreak
+mkdir $LOGSUBDIR
+
+### Initialise the books containing the tasks to do
+book init magicDSToCleanup
+
+### indexes into Database lists
+$magicDSToCleanup_DB = 0
+
+### Check status of tasks
+macro destreak.cleanup.status
+    book listbook magicDSToCleanup
+end
+
+### Reset tasks
+macro destreak.cleanup.reset
+    book init magicDSToCleanup
+end
+
+### Turn tasks on
+macro destreak.cleanup.on
+    task destreak.cleanup.load
+        active true
+    end
+    task destreak.cleanup.run
+        active true
+    end
+end
+
+### Turn tasks off
+macro destreak.cleanup.off
+    task destreak.cleanup.load
+        active false
+    end
+    task destreak.cleanup.run
+        active false
+    end
+end
+
+task	       destreak.cleanup.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 20
+  npending     1
+  active       false
+
+  stdout NULL
+  stderr $LOGSUBDIR/destreak.cleanup.load.log
+
+  task.exec
+    $run = magicdstool -tocleanup
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+
+      # save the DB name for the exit tasks
+      option $DB:$magicDSToCleanup_DB
+      $run = $run -dbname $DB:$magicDSToCleanup_DB
+
+      # only bump database number after we have gone through all of the stages
+      $magicDSToCleanup_DB ++
+      if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook magicDSToCleanup
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup magicDSToCleanup
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       destreak.cleanup.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+  active       false
+
+  task.exec
+    stdout $LOGSUBDIR/destreak.cleanup.run.log
+    stderr $LOGSUBDIR/destreak.cleanup.run.log
+
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages magicDSToCleanup -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images (pantaskState == INIT)
+    book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword magicDSToCleanup $pageName pantaskState RUN
+    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
+    book getword magicDSToCleanup $pageName camera -var CAMERA
+    book getword magicDSToCleanup $pageName stage -var STAGE
+    book getword magicDSToCleanup $pageName outroot -var OUTROOT
+    book getword magicDSToCleanup $pageName dbname -var DBNAME
+
+    sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
+
+    host anyhost
+
+    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
+
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    # if we are ready to run, drop the retry timeout low so we fill up the queue
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    0
+    process_exit magicDSToCleanup $options:0 $JOB_STATUS
+   end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+    process_exit magicDSToCleanup $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword magicDSToCleanup $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
+  end
+end
Index: /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro	(revision 29515)
@@ -10,5 +10,4 @@
 book init magicToDS
 book init magicDSToRevert
-book init magicDSToCleanup
 
 ### indexes into Database lists
@@ -30,13 +29,17 @@
 $magicDSRevertStage = 0
 
+macro destreak.show.stages
+    echo $DS_STAGE:n stages enabled for destreak processing
+    for i 0 $DS_STAGE:n
+        echo $i $DS_STAGE:$i
+    end
+    echo current magicDSStage = $magicDSStage
+    echo current magicDSRevertStage = $magicDSRevertStage
+end
+
 ### Check status of tasks
 macro destreak.status
-    # since echo and listbook output go to different files these do not come in order
-    echo magicToDS
     book listbook magicToDS
-    echo magicDSToRevert
     book listbook magicDSToRevert
-    echo magicDSToCleanup
-    book listbook magicDSToCleanup
 end
 
@@ -45,5 +48,4 @@
     book init magicToDS
     book init magicDSToRevert
-    book init magicDSToCleanup
 end
 
@@ -73,13 +75,4 @@
 end
 
-macro destreak.cleanup.on
-    task destreak.cleanup.load
-        active true
-    end
-    task destreak.cleanup.run
-        active true
-    end
-end
-
 ### Turn tasks off
 macro destreak.off
@@ -106,13 +99,4 @@
     end
 end
-macro destreak.cleanup.off
-    task destreak.cleanup.load
-        active false
-    end
-    task destreak.cleanup.run
-        active false
-    end
-end
-
 
 task	       destreak.load
@@ -203,5 +187,7 @@
     book getword magicToDS $pageName camera -var CAMERA
     book getword magicToDS $pageName streaks_uri -var STREAKS
+    book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE
     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
+    book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE
     book getword magicToDS $pageName stage -var STAGE
     book getword magicToDS $pageName stage_id -var STAGE_ID
@@ -229,5 +215,5 @@
     # TODO: do not add recoveryroot or replace if they are null or zero
 
-    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
 
     add_standard_args run
@@ -274,7 +260,6 @@
 
   periods      -poll $LOADPOLL
-  #periods      -exec $LOADEXEC
   periods      -exec 30
-  periods      -timeout 60
+  periods      -timeout 300
   npending     1
 
@@ -293,4 +278,6 @@
     add_poll_args run
     add_poll_labels run
+    # since this command can be expensive, reduce the limit
+    $run = $run -limit 24
     command $run
   end
@@ -317,8 +304,7 @@
 task	       destreak.revert.load
   host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
+  periods      -poll 60.0
+  periods      -exec 120.
+  periods      -timeout 120
   npending     1
   active       false
@@ -518,131 +504,2 @@
 end
 
-task	       destreak.cleanup.load
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 20
-  npending     1
-  active       false
-
-  stdout NULL
-  stderr $LOGSUBDIR/destreak.cleanup.load.log
-
-  task.exec
-    $run = magicdstool -tocleanup
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-
-      # save the DB name for the exit tasks
-      option $DB:$magicDSToCleanup_DB
-      $run = $run -dbname $DB:$magicDSToCleanup_DB
-
-      # only bump database number after we have gone through all of the stages
-      $magicDSToCleanup_DB ++
-      if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0
-    end
-    add_poll_args run
-    add_poll_labels run
-    command $run
-  end
-
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook magicDSToCleanup
-    end
-
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicDSToCleanup
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
-task	       destreak.cleanup.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       false
-
-  task.exec
-    stdout $LOGSUBDIR/destreak.cleanup.run.log
-    stderr $LOGSUBDIR/destreak.cleanup.run.log
-
-    # if we are unable to run the 'exec', use a long retry time
-    periods -exec $RUNEXEC
-
-    book npages magicDSToCleanup -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    
-    # look for new images (pantaskState == INIT)
-    book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-
-    book setword magicDSToCleanup $pageName pantaskState RUN
-    book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID
-    book getword magicDSToCleanup $pageName camera -var CAMERA
-    book getword magicDSToCleanup $pageName stage -var STAGE
-    book getword magicDSToCleanup $pageName outroot -var OUTROOT
-    book getword magicDSToCleanup $pageName dbname -var DBNAME
-
-    sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID
-
-    host anyhost
-
-    $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile
-
-    add_standard_args run
-
-    # save the pageName for future reference below
-    options $pageName
-
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    # if we are ready to run, drop the retry timeout low so we fill up the queue
-    periods -exec 0.05
-    command $run
-  end
-
-  # default exit status
-  task.exit    0
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-   end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-    process_exit magicDSToCleanup $options:0 $JOB_STATUS
-  end
-
-  task.exit    crash
-    showcommand crash
-    book setword magicDSToCleanup $options:0 pantaskState CRASH
-  end
-
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword magicDSToCleanup $options:0 pantaskState TIMEOUT
-  end
-end
-
Index: /branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTasks/diskbalance.pro	(revision 29515)
@@ -0,0 +1,229 @@
+## diskbalance.pro : tasks for data replication : -*- sh -*-
+## this file contains the tasks for maintaining duplicates as needed in Nebulous
+## these tasks use the books balancePending
+
+# test for required global variables
+check.globals
+
+# load in the local configuration (NEB_USER, etc.)
+module nebulous.site.pro
+
+$LOGSUBDIR = $LOGDIR/balance
+mkdir $LOGSUBDIR
+
+book init balancePending
+book init balanceControl
+
+macro balance.reset
+  book init balancePending
+end
+
+macro balance.status
+  book listbook balancePending
+end
+
+macro balance.on
+  task balance.govern
+    active true
+  end
+  task balance.load
+    active true
+  end
+  task balance.run
+    active true
+  end
+end
+
+macro balance.off
+  task balance.govern
+    active false
+  end
+  task balance.load
+    active false
+  end
+  task balance.run
+    active false
+  end
+end
+
+macro set.host.for.balance
+  if ($0 != 2)
+    echo "USAGE: set.host.for.balance (hostname)"
+    break
+  end
+
+  if (not($PARALLEL))
+    host local
+    return
+  end
+
+# parse volume name
+
+  if ("$1" == "NULL")
+    host anyhost
+  else
+    host $1
+  end
+end
+
+macro set.balance.numbers
+  if ($0 != 3)
+    echo "USAGE: set.balance.numbers (N_sources) (M_destinations)"
+    break
+  end
+
+  book newpage balanceControl control
+  book setword balanceControl control nSources $1
+  book setword balanceControl control mDestinations $2
+end
+
+# the balance process interacts with only the single Nebulous server
+
+# Each 'pendingbalance query is limited to a finite number of so_id values.
+# Each time we call balance.load, we increment the so_id start by the range value
+# If the pendingbalance query exits with exit status 10, we start over at 0
+
+$BALANCE_SO_ID_START = 0
+$BALANCE_SO_ID_RANGE = 500000
+
+# Select Nebulous objects which should be shuffled
+task           balance.load
+  host         local
+
+  periods      -poll 0.5
+  periods      -exec 5
+  periods      -timeout 1500
+  npending     1
+
+  # logs
+  stdout NULL
+  stderr $LOGSUBDIR/balance.log
+    
+  task.exec
+    book npages balancePending -var N
+    if ($N > 2000)
+      process_cleanup balancePending
+      break
+    end
+
+    book getword balanceControl control nSources -var BALANCE_N
+    book getword balanceControl control mDestinations -var BALANCE_M
+
+    command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingbalance --limit 500 --so_id_start $BALANCE_SO_ID_START --so_id_range $BALANCE_SO_ID_RANGE  --balancesources $BALANCE_N --balancedestination $BALANCE_M
+
+  end
+
+   # success : 0 -- we did not hit the limit, advance so_id counter
+  task.exit 0
+    # advance the so_id counter
+    $BALANCE_SO_ID_START = $BALANCE_SO_ID_START + $BALANCE_SO_ID_RANGE
+
+    # convert 'stdout' to book format
+    ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT
+
+    if ($VERBOSE > 2)
+      book listbook balancePending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup balancePending
+  end
+
+  # success : 1 -- we DID hit the limit, do NOT advance so_id counter 
+  task.exit 1
+    # convert 'stdout' to book format
+    ipptool2book stdout balancePending -key key -uniq -setword pantaskState INIT
+
+    if ($VERBOSE > 2)
+      book listbook balancePending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup balancePending
+  end
+
+  # out of so_id range, reset
+  task.exit 10
+    # advance the so_id counter
+    $BALANCE_SO_ID_START = 0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# task to execute shuffles
+task            balance.run
+  periods       -poll 0.5
+  periods       -exec 5
+  periods       -timeout 30
+
+  task.exec
+    book npages balancePending -var N
+    if ($NETWORK == 0) break
+    if ($N == 0)
+        periods -exec 5
+        break
+    end
+    periods -exec 0.05
+
+    # look for new objects in balancePending
+    book getpage balancePending 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setwork balancePending $pageName pantaskState RUN
+
+    book getword balancePending $pageName key              -var KEY
+    book getword balancePending $pageName source_name      -var SOURCE
+    book getword balancePending $pageName destination_name -var DESTINATION
+    
+    # Fix this with multihost restriction when possible.
+    set.host.for.balance $DESTINATION
+
+    stdout NULL
+    stderr $LOGSUBDIR/balance.log
+
+    $run = neb-shift --volume $DESTINATION $KEY $SOURCE
+    
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+	echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit default
+    process_exit balancePending $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword balancePending $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword balancePending $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+
+    
+
Index: /branches/eam_branches/ipp-20100823/ippTasks/dist.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/dist.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/dist.pro	(revision 29515)
@@ -176,7 +176,7 @@
   host         local
 
-  periods      -poll $LOADPOLL
+  periods      -poll 20
   periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -timeout 300
   npending     1
 
@@ -338,6 +338,6 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -exec 30
+  periods      -timeout 3000
   npending     1
 
@@ -479,5 +479,4 @@
     end
     add_poll_labels run
-    echo $run
     command $run
   end
Index: /branches/eam_branches/ipp-20100823/ippTasks/magic.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/magic.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/magic.pro	(revision 29515)
@@ -288,4 +288,6 @@
     book getword magicToProcess $pageName camera -var CAMERA
     book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
+    book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE
+    book getword magicToProcess $pageName exp_name -var EXP_NAME
     book getword magicToProcess $pageName dbname -var DBNAME
 
@@ -308,4 +310,15 @@
     sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
 
+    if ("$NODE" == "root")
+        # This is the root node, put the streaks file into the rawExps workdir
+        # which is presumably in nebulous and thus will be replicatable
+        set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR
+        sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID
+
+        $EXTRA_ARGS = --final-outroot $final_outroot
+    else
+        $EXTRA_ARGS = ""
+    end
+
     $logfile = $baseroot.$NODE.log
     if ("$logfile" == "") 
@@ -319,5 +332,5 @@
     mkdir $outpath
 
-    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile
+    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS
     add_standard_args run
 
Index: /branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/nightly_stacks.pro	(revision 29515)
@@ -436,8 +436,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -498,8 +500,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -560,8 +564,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -623,8 +629,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -684,8 +692,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -747,8 +757,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
@@ -870,8 +882,10 @@
 
     book getword nsData $options:0 nsNmacros -var ns_Nmacros
-    for i 0 $ns_Nmacros
-	sprintf macroName "ns%dMacro" $i
-	book getword nsData $options:0 $macroName -var macroCmd
-	$macroCmd
+    if ("$ns_Nmacros" != "NULL") 
+	for i 0 $ns_Nmacros
+	    sprintf macroName "ns%dMacro" $i
+	    book getword nsData $options:0 $macroName -var macroCmd
+	    $macroCmd
+	end
     end
 
Index: /branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/pstamp.pro	(revision 29515)
@@ -14,5 +14,7 @@
 $pstampRev_DB = 0
 $pstampDep_DB = 0
+$pstampRevDep_DB = 0
 $pstampCleanup_DB = 0
+$pstampStopFaulted_DB = 0
 
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
@@ -89,7 +91,4 @@
         active false
     end
-    task pstamp.job.revert
-        active false
-    end
     task pstamp.dependent.load
         active false
@@ -104,7 +103,19 @@
         active true
     end
+    task pstamp.dependent.revert
+        active true
+    end
+    task pstamp.stopfaulted
+        active true
+    end
 end
 macro pstamp.revert.off
     task pstamp.job.revert
+        active false
+    end
+    task pstamp.dependent.revert
+        active false
+    end
+    task pstamp.stopfaulted
         active false
     end
@@ -513,5 +524,5 @@
         stdout NULL
         stderr $LOGSUBDIR/pstamp.job.revert.log
-        $run = pstamptool -revertjob -all
+        $run = pstamptool -revertjob
         if ($DB:n == 0)
             option DEFAULT
@@ -798,2 +809,83 @@
 end
 
+task pstamp.dependent.revert
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec 900
+    periods     -timeout 300
+    npending    1
+
+    task.exec
+        stdout NULL
+        stderr $LOGSUBDIR/pstamp.dependent.revert.log
+        $run = pstamptool -revertdependent
+        if ($DB:n == 0)
+            option DEFAULT
+        else 
+            $run = $run $PS_DBSERVER -dbname $DB:$pstampRevDep_DB
+            $pstampRevDep_DB ++
+            if ($pstampRevDep_DB >= $DB:n) set pstampRevDep_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end
+
+task pstamp.stopfaulted
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec 30
+    periods     -timeout 300
+    npending    1
+
+    task.exec
+        stderr $LOGSUBDIR/pstamp.stopfaulted.log
+        stderr $LOGSUBDIR/pstamp.stopfaulted.log
+        $run = pstampstopfaulted
+        if ($DB:n == 0)
+            option DEFAULT
+        else 
+            $run = $run $PS_DBSERVER -dbname $DB:$pstampStopFaulted_DB
+            $pstampStopFaulted_DB ++
+            if ($pstampStopFaulted_DB >= $DB:n) set pstampStopFaulted_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end
Index: /branches/eam_branches/ipp-20100823/ippTasks/publish.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/publish.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/publish.pro	(revision 29515)
@@ -75,4 +75,6 @@
 
   task.exec
+    if ($LABEL:n == 0) break
+    #otherwise everything will be queued when there is no label.
     $run = pubtool -definerun
     if ($DB:n == 0)
Index: /branches/eam_branches/ipp-20100823/ippTasks/replicate.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/replicate.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/replicate.pro	(revision 29515)
@@ -173,4 +173,5 @@
     book getword replicatePending $pageName volume_name -var VOLUME_NAME
     book getword replicatePending $pageName volume_host -var VOLUME_HOST
+    book getword replicatePending $pageName command     -var COMMAND
 
     set.host.for.replicate $VOLUME_HOST
@@ -180,5 +181,5 @@
 
     # these operations do not require a database to be specified
-    $run = neb-replicate --copies $NEED_COPIES $KEY
+    $run = $COMMAND $KEY
 
     # save the pageName for future reference below
Index: /branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTasks/science.cleanup.pro	(revision 29515)
@@ -113,5 +113,8 @@
   task.exec
     book npages chipPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -143,4 +146,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -569,5 +573,8 @@
   task.exec
     book npages warpPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -599,4 +606,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -721,5 +729,8 @@
   task.exec
     book npages diffCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -751,4 +762,5 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
Index: /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/cmfExtensions.txt
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/cmfExtensions.txt	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/cmfExtensions.txt	(revision 29515)
@@ -0,0 +1,370 @@
+
+##### SkyChip.hdr ####
+
+IMPLE  =                    T / file does conform to FITS standard
+BITPIX  =                   16 / number of bits per data pixel
+NAXIS   =                    0 / number of data axes
+EXTEND  =                    T / FITS dataset may contain extensions
+COMMENT   FITS (Flexible Image Transport System) format is defined in 'Astronomy
+COMMENT   and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H
+PSCAMERA= 'GPC1    '           / Camera name
+PSFORMAT= 'SKYCELL '           / Camera format
+HIERARCH FPA.TELESCOPE = 'PS1     ' / Telescope of origin
+HIERARCH FPA.INSTRUMENT = 'GPC1    ' / Instrument name (according to the instrum
+HIERARCH FPA.DETECTOR = 'GPC1    ' / Detector name
+HIERARCH FPA.COMMENT = '        ' / Observation comment
+HIERARCH FPA.OBS.MODE = '        ' / Observation mode (eg, survey id)
+HIERARCH FPA.OBS.GROUP = '        ' / Observation group (eg, associated images)
+HIERARCH FPA.FOCUS = 'NaN     ' / Telescope focus
+AIRMASS = 'NaN     '           / Airmass at boresight
+HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
+HIERARCH FPA.FILTER = 'r.00000 ' / Filter used (instrument name)
+HIERARCH FPA.POSANGLE = 'NaN     ' / Position angle of instrument
+HIERARCH FPA.ROTANGLE = 'NaN     ' / Rotator angle of instrument
+HIERARCH FPA.RADECSYS = '        ' / Celestial coordinate system
+HIERARCH FPA.RA = 'NaN     '   / Right Ascension of boresight
+HIERARCH FPA.DEC = 'NaN     '  / Declination of boresight
+HIERARCH FPA.LONGITUDE = 'NaN     ' / West longitude of observatory
+HIERARCH FPA.LATITUDE = 'NaN     ' / Latitude of observatory
+HIERARCH FPA.ELEVATION = 'NaN     ' / Elevation of observatory (meters)
+HIERARCH FPA.OBSTYPE = '        ' / Type of observation
+HIERARCH FPA.OBJECT = '        ' / Object of observation
+HIERARCH FPA.ALT = 'NaN     '  / Altitude of boresight
+HIERARCH FPA.AZ = 'NaN     '   / Azimuth of boresight
+HIERARCH FPA.TEMP = 'NaN     ' / Temperature of focal plane
+HIERARCH FPA.M1X = 'NaN     '  / Primary Mirror X Position
+HIERARCH FPA.M1Y = 'NaN     '  / Primary Mirror Y Position
+HIERARCH FPA.M1Z = 'NaN     '  / Primary Mirror Z Position
+HIERARCH FPA.M1TIP = 'NaN     ' / Primary Mirror TIP
+HIERARCH FPA.M1TILT = 'NaN     ' / Primary Mirror TILT
+HIERARCH FPA.M2X = 'NaN     '  / Primary Mirror X Position
+HIERARCH FPA.M2Y = 'NaN     '  / Primary Mirror Y Position
+HIERARCH FPA.M2Z = 'NaN     '  / Primary Mirror Z Position
+HIERARCH FPA.M2TIP = 'NaN     ' / Primary Mirror TIP
+HIERARCH FPA.M2TILT = 'NaN     ' / Primary Mirror TILT
+HIERARCH FPA.ENV.TEMP = 'NaN     ' / Environment: Temperature
+HIERARCH FPA.ENV.HUMID = 'NaN     ' / Environment: Humidity
+HIERARCH FPA.ENV.WIND = 'NaN     ' / Environment: Wind speed
+HIERARCH FPA.ENV.DIR = 'NaN     ' / Environment: Wind direction
+HIERARCH FPA.TELTEMP.M1 = 'NaN     ' / Telescope Temperatures: M1
+HIERARCH FPA.TELTEMP.M1CELL = 'NaN     ' / Telescope Temperatures: M1 cell
+HIERARCH FPA.TELTEMP.M2 = 'NaN     ' / Telescope Temperatures: M2
+HIERARCH FPA.TELTEMP.SPIDER = 'NaN     ' / Telescope Temperatures: spider
+HIERARCH FPA.TELTEMP.TRUSS = 'NaN     ' / Telescope Temperatures: truss
+HIERARCH FPA.TELTEMP.EXTRA = 'NaN     ' / Telescope Temperatures: extra
+HIERARCH FPA.PON.TIME = 'NaN     ' / Power On Time
+HIERARCH FPA.ZP =          25. / Magnitude zero point
+HIERARCH CHIP.XSIZE =        0 / Size of chip (pixels)
+HIERARCH CHIP.YSIZE =        0 / Size of chip (pixels)
+HIERARCH CHIP.TEMP = 'NaN     ' / Temperature of chip
+HIERARCH CHIP.TEMPERATURE = 'NaN     ' / Temperature of chip
+HIERARCH CHIP.ID = '        '  / Chip identifier
+HIERARCH CHIP.SEEING = 12.07651 / Seeing FWHM (pixels)
+HIERARCH CELL.GAIN =  1.029656 / CCD gain (e/count)
+HIERARCH CELL.READNOISE = 7.715189 / CCD read noise (e)
+HIERARCH CELL.SATURATION = 14000. / Saturation level (counts)
+HIERARCH CELL.BAD =         0. / Bad level (counts)
+EXPTIME =               14626. / Exposure time (sec)
+HIERARCH CELL.DARKTIME = 'NaN     ' / Time since flush (sec)
+HIERARCH CELL.XBIN =         1 / Binning in x
+HIERARCH CELL.YBIN =         1 / Binning in y
+MJD-OBS =     55214.1819563957 / Time of exposure
+HIERARCH CELL.XSIZE =        0 / Size of cell (pixels)
+HIERARCH CELL.YSIZE =        0 / Size of cell (pixels)
+HIERARCH CELL.XWINDOW =      0 / Start of cell window (pixels)
+HIERARCH CELL.YWINDOW =      0 / Start of cell window (pixels)
+HIERARCH CELL.TRIMSEC = '[nan:nan,nan:nan]' / Trim section
+HIERARCH CELL.BIASSEC = ''
+EXTDATA = 'SkyChip.psf'        / name of table extension
+EXTTYPE = 'IMAGE   '           / extension type
+PHOTCODE= 'GPC1.r.SkyChip'     / photcode from FPA concepts
+APMIFIT =           -0.2959032 / aperture residual
+DAPMIFIT=           0.08735527 / ap residual scatter
+NAPMIFIT=                  960 / number of apresid stars
+NPSFSTAR=                  249 / Number of stars used to make PSF
+APLOSS  =            0.2532406 / aperture loss (mag)
+FWHM_MAJ=             12.12506 / PSF FWHM Major axis (mean)
+FW_MJ_SG=                   0. / PSF FWHM Major axis (sigma)
+FW_MJ_LQ=             12.12506 / PSF FWHM Major axis (lower quartile)
+FW_MJ_UQ=             12.12506 / PSF FWHM Major axis (upper quartile)
+FWHM_MIN=             12.02795 / PSF FWHM Minor axis (mean)
+FW_MN_SG=                   0. / PSF FWHM Minor axis (sigma)
+FW_MN_LQ=             12.02795 / PSF FWHM Minor axis (lower quartile)
+FW_MN_UQ=             12.02795 / PSF FWHM Minor axis (upper quartile)
+ANGLE   =             1.043148 / PSF angle
+IQ_NSTAR=                 3096 / Number of stars used for IQ measurements
+IQ_FW1  =             7.789053 / FWHM of Major Axis from moments
+IQ_FW1_E=            0.6052215 / FWHM scatter (Major) from moments
+IQ_FW2  =             7.137493 / FWHM of Minor Axis from moments
+IQ_FW2_E=            0.5436327 / FWHM scatter (Minor) from moments
+IQ_M2   =             1.766379 / M_2 = sqrt (M_c2^2 + M_s2^2)
+IQ_M2_ER=             1.457713 / Stdev of M_2
+IQ_M2_LQ=            0.6377338 / Lower Quartile of M_2
+IQ_M2_UQ=             2.536107 / Upper Quartile of M_2
+IQ_M2C  =           0.03108882 / M_2c = sum f r^2 cos(2phi) / sum f
+IQ_M2C_E=              1.49958 / Stdev of M_2c
+IQ_M2C_L=           -0.7241516 / Lower Quartile of M_2c
+IQ_M2C_U=            0.7717791 / Upper Quartile of M_2c
+IQ_M2S  =           0.05257827 / M_2s = sum f r^2 cos(2phi) / sum f
+IQ_M2S_E=             1.730192 / Stdev of M_2s
+IQ_M2S_L=           -0.7311587 / Lower Quartile of M_2s
+IQ_M2S_U=            0.7727656 / Upper Quartile of M_2s
+IQ_M3   =            0.9306768 / M_3 = sqrt (M_c3^2 + M_s3^2)
+IQ_M3_ER=             0.866469 / Stdev of M_3
+IQ_M3_LQ=            0.2926856 / Lower Quartile of M_3
+IQ_M3_UQ=             1.309593 / Upper Quartile of M_3
+IQ_M4   =           0.07706664 / M_4 = sqrt (M_c4^2 + M_s4^2)
+IQ_M4_ER=           0.06705837 / Stdev of M_4
+IQ_M4_LQ=           0.02815639 / Lower Quartile of M_4
+IQ_M4_UQ=            0.1044456 / Upper Quartile of M_4
+FSATUR  =                   0. / SATURATION MAG
+FLIMIT  =                   0. / COMPLETENESS MAG
+DT_PHOT =             378.8021 / elapsed psphot time
+NSTARS  =                10884 / Number of sources
+NDET_EXT=                 3068 / Number of extended sources
+NDET_CR =                   27 / Number of cosmic rays
+IMNAXIS1=                 4651 / Number of columns in original image
+IMNAXIS2=                 4651 / Number of rows in original image
+EXTNAME = 'SkyChip.hdr'
+END
+
+#### SkyChip.psf ####
+
+XTENSION= 'BINTABLE'           / binary table extension
+BITPIX  =                    8 / 8-bit bytes
+NAXIS   =                    2 / 2-dimensional binary table
+NAXIS1  =                  376 / width of table in bytes
+NAXIS2  =                 5553 / number of rows in table
+PCOUNT  =                    0 / size of special data area
+GCOUNT  =                    1 / one data group (required keyword)
+TFIELDS =                   51 / number of fields in each row
+TTYPE1  = 'IPP_IDET'           / label for field   1
+TFORM1  = '1J      '           / data format of field: 4-byte INTEGER
+TZERO1  =           2147483648 / offset for unsigned integers
+TSCAL1  =                    1 / data are not scaled
+TTYPE2  = 'X_PSF   '           / label for field   2
+TFORM2  = '1E      '           / data format of field: 4-byte REAL
+TTYPE3  = 'Y_PSF   '           / label for field   3
+TFORM3  = '1E      '           / data format of field: 4-byte REAL
+TTYPE4  = 'X_PSF_SIG'          / label for field   4
+TFORM4  = '1E      '           / data format of field: 4-byte REAL
+TTYPE5  = 'Y_PSF_SIG'          / label for field   5
+TFORM5  = '1E      '           / data format of field: 4-byte REAL
+TTYPE6  = 'POSANGLE'           / label for field   6
+TFORM6  = '1E      '           / data format of field: 4-byte REAL
+TTYPE7  = 'PLTSCALE'           / label for field   7
+TFORM7  = '1E      '           / data format of field: 4-byte REAL
+TTYPE8  = 'PSF_INST_MAG'       / label for field   8
+TFORM8  = '1E      '           / data format of field: 4-byte REAL
+TTYPE9  = 'PSF_INST_MAG_SIG'   / label for field   9
+TFORM9  = '1E      '           / data format of field: 4-byte REAL
+TTYPE10 = 'PSF_INST_FLUX'      / label for field  10
+TFORM10 = '1E      '           / data format of field: 4-byte REAL
+TTYPE11 = 'PSF_INST_FLUX_SIG'  / label for field  11
+TFORM11 = '1E      '           / data format of field: 4-byte REAL
+TTYPE12 = 'AP_MAG  '           / label for field  12
+TFORM12 = '1E      '           / data format of field: 4-byte REAL
+TTYPE13 = 'AP_MAG_RAW'         / label for field  13
+TFORM13 = '1E      '           / data format of field: 4-byte REAL
+TTYPE14 = 'AP_MAG_RADIUS'      / label for field  14
+TFORM14 = '1E      '           / data format of field: 4-byte REAL
+TTYPE15 = 'PEAK_FLUX_AS_MAG'   / label for field  15
+TFORM15 = '1E      '           / data format of field: 4-byte REAL
+TTYPE16 = 'CAL_PSF_MAG'        / label for field  16
+TFORM16 = '1E      '           / data format of field: 4-byte REAL
+TTYPE17 = 'CAL_PSF_MAG_SIG'    / label for field  17
+TFORM17 = '1E      '           / data format of field: 4-byte REAL
+TTYPE18 = 'RA_PSF  '           / label for field  18
+TFORM18 = '1D      '           / data format of field: 8-byte DOUBLE
+TTYPE19 = 'DEC_PSF '           / label for field  19
+TFORM19 = '1D      '           / data format of field: 8-byte DOUBLE
+TTYPE20 = 'SKY     '           / label for field  20
+TFORM20 = '1E      '           / data format of field: 4-byte REAL
+TTYPE21 = 'SKY_SIGMA'          / label for field  21
+TFORM21 = '1E      '           / data format of field: 4-byte REAL
+TTYPE22 = 'PSF_CHISQ'          / label for field  22
+TFORM22 = '1E      '           / data format of field: 4-byte REAL
+TTYPE23 = 'CR_NSIGMA'          / label for field  23
+TFORM23 = '1E      '           / data format of field: 4-byte REAL
+TTYPE24 = 'EXT_NSIGMA'         / label for field  24
+TFORM24 = '1E      '           / data format of field: 4-byte REAL
+TTYPE25 = 'PSF_MAJOR'          / label for field  25
+TFORM25 = '1E      '           / data format of field: 4-byte REAL
+TTYPE26 = 'PSF_MINOR'          / label for field  26
+TFORM26 = '1E      '           / data format of field: 4-byte REAL
+TTYPE27 = 'PSF_THETA'          / label for field  27
+TFORM27 = '1E      '           / data format of field: 4-byte REAL
+TTYPE28 = 'PSF_QF  '           / label for field  28
+TFORM28 = '1E      '           / data format of field: 4-byte REAL
+TTYPE29 = 'PSF_QF_PERFECT'     / label for field  29
+TFORM29 = '1E      '           / data format of field: 4-byte REAL
+TTYPE30 = 'PSF_NDOF'           / label for field  30
+TFORM30 = '1J      '           / data format of field: 4-byte INTEGER
+TTYPE31 = 'PSF_NPIX'           / label for field  31
+TFORM31 = '1J      '           / data format of field: 4-byte INTEGER
+TTYPE32 = 'MOMENTS_XX'         / label for field  32
+TFORM32 = '1E      '           / data format of field: 4-byte REAL
+TTYPE33 = 'MOMENTS_XY'         / label for field  33
+TFORM33 = '1E      '           / data format of field: 4-byte REAL
+TTYPE34 = 'MOMENTS_YY'         / label for field  34
+TFORM34 = '1E      '           / data format of field: 4-byte REAL
+TTYPE35 = 'MOMENTS_M3C'        / label for field  35
+TFORM35 = '1E      '           / data format of field: 4-byte REAL
+TTYPE36 = 'MOMENTS_M3S'        / label for field  36
+TFORM36 = '1E      '           / data format of field: 4-byte REAL
+TTYPE37 = 'MOMENTS_M4C'        / label for field  37
+TFORM37 = '1E      '           / data format of field: 4-byte REAL
+TTYPE38 = 'MOMENTS_M4S'        / label for field  38
+TFORM38 = '1E      '           / data format of field: 4-byte REAL
+TTYPE39 = 'MOMENTS_R1'         / label for field  39
+TFORM39 = '1E      '           / data format of field: 4-byte REAL
+TTYPE40 = 'MOMENTS_RH'         / label for field  40
+TFORM40 = '1E      '           / data format of field: 4-byte REAL
+TTYPE41 = 'KRON_FLUX'          / label for field  41
+TFORM41 = '1E      '           / data format of field: 4-byte REAL
+TTYPE42 = 'KRON_FLUX_ERR'      / label for field  42
+TFORM42 = '1E      '           / data format of field: 4-byte REAL
+TTYPE43 = 'KRON_FLUX_INNER'    / label for field  43
+TFORM43 = '1E      '           / data format of field: 4-byte REAL
+TTYPE44 = 'KRON_FLUX_OUTER'    / label for field  44
+TFORM44 = '1E      '           / data format of field: 4-byte REAL
+TTYPE45 = 'FLAGS   '           / label for field  45
+TFORM45 = '1J      '           / data format of field: 4-byte INTEGER
+TZERO45 =           2147483648 / offset for unsigned integers
+TSCAL45 =                    1 / data are not scaled
+TTYPE46 = 'FLAGS2  '           / label for field  46
+TFORM46 = '1J      '           / data format of field: 4-byte INTEGER
+TZERO46 =           2147483648 / offset for unsigned integers
+TSCAL46 =                    1 / data are not scaled
+TTYPE47 = 'APER_FLUX'          / label for field  47
+TFORM47 = '15E     '           / data format of field: 4-byte REAL
+TTYPE48 = 'APER_FLUX_ERR'      / label for field  48
+TFORM48 = '15E     '           / data format of field: 4-byte REAL
+TTYPE49 = 'APER_FILL'          / label for field  49
+TFORM49 = '15E     '           / data format of field: 4-byte REAL
+TTYPE50 = 'N_FRAMES'           / label for field  50
+TFORM50 = '1I      '           / data format of field: 2-byte INTEGER
+TZERO50 =                32768 / offset for unsigned integers
+TSCAL50 =                    1 / data are not scaled
+TTYPE51 = 'PADDING '           / label for field  51
+TFORM51 = '1I      '           / data format of field: 2-byte INTEGER
+EXTHEAD = 'SkyChip.hdr'        / name of image extension w/
+EXTTYPE = 'PS1_SV1 '           / extension type
+XSRCNAME= 'SkyChip.xsrc'       / name of XSRC table extension
+XFITNAME= 'SkyChip.xfit'       / name of XFIT table extension
+HIERARCH SOURCE.MASK.PSFMODEL = 1 / Fit with PSF model
+HIERARCH SOURCE.MASK.EXTMODEL = 2 / Fit with extended model
+HIERARCH SOURCE.MASK.FITTED = 4 / Fit with non-linear model
+HIERARCH SOURCE.MASK.FAIL =  8 / Non-linear fit failed
+HIERARCH SOURCE.MASK.POOR = 16 / Non-linear fit poor
+HIERARCH SOURCE.MASK.PAIR = 32 / Fit with double PSF
+HIERARCH SOURCE.MASK.PSFSTAR = 64 / Used to define PSF model
+HIERARCH SOURCE.MASK.SATSTAR = 128 / Model peak is above saturation
+HIERARCH SOURCE.MASK.BLEND = 256 / Blended with other sources
+HIERARCH SOURCE.MASK.EXTERNAL = 512 / Based on supplied input position
+HIERARCH SOURCE.MASK.BADPSF = 1024 / Unable to estimate object PSF
+HIERARCH SOURCE.MASK.DEFECT = 2048 / Suspected defect
+HIERARCH SOURCE.MASK.SATURATED = 4096 / Suspected saturated (bleed trail)
+HIERARCH SOURCE.MASK.CR_LIMIT = 8192 / Suspected cosmic ray
+HIERARCH SOURCE.MASK.EXT_LIMIT = 16384 / Suspected extended
+HIERARCH SOURCE.MASK.MOMENTS_FAILURE = 32768 / Failed to measure moments
+HIERARCH SOURCE.MASK.SKY_FAILURE = 65536 / Failed to measure local sky
+HIERARCH SOURCE.MASK.SKYVAR_FAILURE = 131072 / Failed to measure sky variance
+HIERARCH SOURCE.MASK.BELOW_MOMENTS_SN = 262144 / Moments not measured due to low
+HIERARCH SOURCE.MASK.BIG_RADIUS = 1048576 / Small radius has poor moments
+HIERARCH SOURCE.MASK.AP_MAGS = 2097152 / Measured aperture magnitude
+HIERARCH SOURCE.MASK.BLEND_FIT = 4194304 / Fit as a blend
+HIERARCH SOURCE.MASK.EXTENDED_FIT = 8388608 / Fit with full extended fit
+HIERARCH SOURCE.MASK.EXTENDED_STATS = 16777216 / Calculated extended aperture st
+HIERARCH SOURCE.MASK.LINEAR_FIT = 33554432 / Fit with linear fit
+HIERARCH SOURCE.MASK.NONLINEAR_FIT = 67108864 / Fit with non-linear fit
+HIERARCH SOURCE.MASK.RADIAL_FLUX = 134217728 / Calculated radial flux measuremen
+HIERARCH SOURCE.MASK.SIZE_SKIPPED = 268435456 / Could not be determine size
+HIERARCH SOURCE.MASK.ON_SPIKE = 536870912 / Source lands in bright star spike
+HIERARCH SOURCE.MASK.ON_GHOST = 1073741824 / Source lands in bright star ghost /
+HIERARCH SOURCE.MASK.OFF_CHIP = 2147483648 / Source centroid lands off chip edge
+EXTNAME = 'SkyChip.psf'
+END
+
+#### SkyChip.xsrc ####
+
+TENSION= 'BINTABLE'           / binary table extension
+BITPIX  =                    8 / 8-bit bytes
+NAXIS   =                    2 / 2-dimensional binary table
+NAXIS1  =                  240 / width of table in bytes
+NAXIS2  =                  264 / number of rows in table
+PCOUNT  =                    0 / size of special data area
+GCOUNT  =                    1 / one data group (required keyword)
+TFIELDS =                   18 / number of fields in each row
+TTYPE1  = 'IPP_IDET'           / label for field   1
+TFORM1  = '1J      '           / data format of field: 4-byte INTEGER
+TZERO1  =           2147483648 / offset for unsigned integers
+TSCAL1  =                    1 / data are not scaled
+TTYPE2  = 'X_EXT   '           / label for field   2
+TFORM2  = '1E      '           / data format of field: 4-byte REAL
+TTYPE3  = 'Y_EXT   '           / label for field   3
+TFORM3  = '1E      '           / data format of field: 4-byte REAL
+TTYPE4  = 'X_EXT_SIG'          / label for field   4
+TFORM4  = '1E      '           / data format of field: 4-byte REAL
+TTYPE5  = 'Y_EXT_SIG'          / label for field   5
+TFORM5  = '1E      '           / data format of field: 4-byte REAL
+TTYPE6  = 'F25_ARATIO'         / label for field   6
+TFORM6  = '1E      '           / data format of field: 4-byte REAL
+TTYPE7  = 'F25_THETA'          / label for field   7
+TFORM7  = '1E      '           / data format of field: 4-byte REAL
+TTYPE8  = 'PETRO_MAG'          / label for field   8
+TFORM8  = '1E      '           / data format of field: 4-byte REAL
+TTYPE9  = 'PETRO_MAG_ERR'      / label for field   9
+TFORM9  = '1E      '           / data format of field: 4-byte REAL
+TTYPE10 = 'PETRO_RADIUS'       / label for field  10
+TFORM10 = '1E      '           / data format of field: 4-byte REAL
+TTYPE11 = 'PETRO_RADIUS_ERR'   / label for field  11
+TFORM11 = '1E      '           / data format of field: 4-byte REAL
+TTYPE12 = 'PETRO_RADIUS_50'    / label for field  12
+TFORM12 = '1E      '           / data format of field: 4-byte REAL
+TTYPE13 = 'PETRO_RADIUS_50_ERR' / label for field  13
+TFORM13 = '1E      '           / data format of field: 4-byte REAL
+TTYPE14 = 'PETRO_RADIUS_90'    / label for field  14
+TFORM14 = '1E      '           / data format of field: 4-byte REAL
+TTYPE15 = 'PETRO_RADIUS_90_ERR' / label for field  15
+TFORM15 = '1E      '           / data format of field: 4-byte REAL
+TTYPE16 = 'PROF_SB '           / label for field  16
+TFORM16 = '15E     '           / data format of field: 4-byte REAL
+TTYPE17 = 'PROF_FLUX'          / label for field  17
+TFORM17 = '15E     '           / data format of field: 4-byte REAL
+TTYPE18 = 'PROF_FILL'          / label for field  18
+TFORM18 = '15E     '           / data format of field: 4-byte REAL
+RMIN_00 =                   0. / min radius for SB profile
+RMAX_00 =                 0.56 / min radius for SB profile
+RMIN_01 =                 0.56 / min radius for SB profile
+RMAX_01 =                 1.69 / min radius for SB profile
+RMIN_02 =                 1.69 / min radius for SB profile
+RMAX_02 =                 2.59 / min radius for SB profile
+RMIN_03 =                 2.59 / min radius for SB profile
+RMAX_03 =                 4.41 / min radius for SB profile
+RMIN_04 =                 4.41 / min radius for SB profile
+RMAX_04 =                 7.51 / min radius for SB profile
+RMIN_05 =                 7.51 / min radius for SB profile
+RMAX_05 =                11.58 / min radius for SB profile
+RMIN_06 =                11.58 / min radius for SB profile
+RMAX_06 =                18.58 / min radius for SB profile
+RMIN_07 =                18.58 / min radius for SB profile
+RMAX_07 =                28.55 / min radius for SB profile
+RMIN_08 =                28.55 / min radius for SB profile
+RMAX_08 =                 45.5 / min radius for SB profile
+RMIN_09 =                 45.5 / min radius for SB profile
+RMAX_09 =                70.51 / min radius for SB profile
+RMIN_10 =                70.51 / min radius for SB profile
+RMAX_10 =               110.53 / min radius for SB profile
+RMIN_11 =               110.53 / min radius for SB profile
+RMAX_11 =               172.49 / min radius for SB profile
+RMIN_12 =               172.49 / min radius for SB profile
+RMAX_12 =               269.52 / min radius for SB profile
+RMIN_13 =               269.52 / min radius for SB profile
+RMAX_13 =               420.51 / min radius for SB profile
+RMIN_14 =               420.51 / min radius for SB profile
+RMAX_14 =                652.5 / min radius for SB profile
+EXTNAME = 'SkyChip.xsrc'
+END
+
+
Index: /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/map.xml
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/map.xml	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/map.xml	(revision 29515)
@@ -0,0 +1,387 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<tabledata type="stack">
+  <table name="StackMeta" ippfitsextension="SkyChip.hdr">
+    <!--  <map pspsName="stackMetaID" ippType="TLONGLONG" ippName="" comment="stack identifier"/> -->
+    <!--  *** DONE IN CODE <map pspsName="skycellID" ippType="TLONG" ippName="" comment="skycell region identifier"/> -->
+    <!--  *** DONE IN CODE <map pspsName="surveyID" ippType="TBYTE" ippName="" comment="survey flag identifier"/> -->
+    <!--  <map pspsName="photoCalID" ippType="TLONG" ippName="" comment="photometry code numerical id"/> -->
+    <!--  *** DONE IN CODE <map pspsName="filterID" ippType="TBYTE" ippName="" comment="filter identifier"/> -->
+    <!--  <map pspsName="stackTypeID" ippType="TBYTE" ippName="" comment="stack type identifier"/> -->
+    <!--  <map pspsName="refImageID" ippType="TLONGLONG" ippName="" comment="identifier of image used as reference for analysis"/> -->
+    <!--  <map pspsName="subtrImageID" ippType="TLONGLONG" ippName="" comment="identifier of image subtracted to generate difference image"/> -->
+          <map pspsName="magSat" ippType="TFLOAT" ippName="FSATUR" comment="saturation magnitude level"/>
+    <!--  <map pspsName="analVer" ippType="TSHORT" ippName="" comment="analysis version index"/> -->
+    <!--  <map pspsName="nP2Images" ippType="TSHORT" ippName="" comment="number of P2 images contributing to this cell"/> -->
+          <map pspsName="completMag" ippType="TFLOAT" ippName="FLIMIT" comment="95% completion level in mag"/>
+    <!--  <map pspsName="astroScat" ippType="TFLOAT" ippName="" comment="astrometric scatter for chip"/> -->
+    <!--  <map pspsName="photoScat" ippType="TFLOAT" ippName="" comment="photometric scatter for chip"/> -->
+    <!--  <map pspsName="nAstroRef" ippType="TLONG" ippName="" comment="number of astrometric reference sources"/> -->
+    <!--  <map pspsName="nPhoRef" ippType="TLONG" ippName="" comment="number of photometric reference sources"/> -->
+    <!--  <map pspsName="psfFwhm" ippType="TFLOAT" ippName="" comment="PSF full width at half maximum"/> -->
+    <!--  <map pspsName="psfmodelID" ippType="TSHORT" ippName="" comment="PSF model identifier"/> -->
+          <map pspsName="psfWidMajor" ippType="TFLOAT" ippName="FWHM_MAJ" comment="PSF parameters"/> 
+          <map pspsName="psfWidMinor" ippType="TFLOAT" ippName="FWHM_MIN" comment="PSF parameters"/>
+          <map pspsName="psfTheta" ippType="TFLOAT" ippName="ANGLE" comment="PSF parameters"/> 
+    <!--  <map pspsName="psfExtra1" ippType="TFLOAT" ippName="" comment="PSF parameters"/> -->
+    <!--  <map pspsName="psfExtra2" ippType="TFLOAT" ippName="" comment="PSF parameters"/> -->
+    <!--  <map pspsName="photoZero" ippType="TFLOAT" ippName="" comment="local derived photometric zero point"/> -->
+    <!--  <map pspsName="photoColor" ippType="TFLOAT" ippName="" comment="local derived photometric color term"/> -->
+    <!--  <map pspsName="ctype1" ippType="TSTRING" ippName="" comment="name of astrometric projection in RA"/> -->
+    <!--  <map pspsName="ctype2" ippType="TSTRING" ippName="" comment="name of astrometric projection in DEC"/> -->
+    <!--  <map pspsName="crval1" ippType="TDOUBLE" ippName="" comment="RA corresponding to reference pixel"/> -->
+    <!--  <map pspsName="crval2" ippType="TDOUBLE" ippName="" comment="DEC corresponding to reference pixel"/> -->
+    <!--  <map pspsName="crpix1" ippType="TDOUBLE" ippName="" comment="reference pixel value for RA"/> -->
+    <!--  <map pspsName="crpix2" ippType="TDOUBLE" ippName="" comment="reference pixel value for DEC"/> -->
+    <!--  <map pspsName="cdelt1" ippType="TDOUBLE" ippName="" comment="scale factor for RA"/> -->
+    <!--  <map pspsName="cdelt2" ippType="TDOUBLE" ippName="" comment="scale factor for DEC"/> -->
+    <!--  <map pspsName="pc001001" ippType="TDOUBLE" ippName="" comment="elements of rotation/Dcale matrix"/> -->
+    <!--  <map pspsName="pc001002" ippType="TDOUBLE" ippName="" comment="elements of rotation/Dcale matrix"/> -->
+    <!--  <map pspsName="pc002001" ippType="TDOUBLE" ippName="" comment="elements of rotation/Dcale matrix"/> -->
+    <!--  <map pspsName="pc002002" ippType="TDOUBLE" ippName="" comment="elements of rotation/Dcale matrix"/> -->
+    <!--  <map pspsName="polyOrder" ippType="TBYTE" ippName="" comment="polynomial order of astrometry fit"/> -->
+    <!--  <map pspsName="pca1x3y0" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x2y1" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x1y2" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x0y3" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x2y0" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x1y1" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca1x0y2" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x3y0" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x2y1" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x1y2" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x0y3" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x2y0" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x1y1" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="pca2x0y2" ippType="TDOUBLE" ippName="" comment="polynomial coefficients for the astrometric fit"/> -->
+    <!--  <map pspsName="calibModNum" ippType="TSHORT" ippName="" comment="calibration modification number"/> -->
+    <!--  <map pspsName="dataRelease" ippType="TBYTE" ippName="" comment="Data release"/> -->
+  </table>
+  <table name="StackDetection" ippfitsextension="SkyChip.psf">
+    <!--  <map pspsName="objID" ippType="TLONGLONG" ippName="" comment="ODM object identifier"/> -->
+    <!--  <map pspsName="stackDetectID" ippType="TLONGLONG" ippName="" comment="ODM detection identifier"/> -->
+    <!--  <map pspsName="ippObjID" ippType="TLONGLONG" ippName="" comment="IPP object identifier"/> -->
+          <map pspsName="ippDetectID" ippType="TLONGLONG" ippName="IPP_IDET" comment="detection ID generated by IPP"/> 
+    <!--  <map pspsName="filterID" ippType="TBYTE" ippName="" comment="filter identifier"/> -->
+    <!--  <map pspsName="stackTypeID" ippType="TBYTE" ippName="" comment="stack type identifier"/> -->
+    <!--   *** DONE IN CODE <map pspsName="surveyID" ippType="TBYTE" ippName="" comment="survey flag identifier"/> -->
+    <!--  <map pspsName="primaryF" ippType="TBYTE" ippName="" comment="identifies best stack detection for Stacks overlapping the same region of the sky."/> -->
+    <!--  <map pspsName="stackMetaID" ippType="TLONGLONG" ippName="" comment="stack identifier"/> -->
+    <!--  *** DONE IN CODE <map pspsName="skyCellID" ippType="TLONG" ippName="" comment="skycell identifier"/> -->
+    <!--  <map pspsName="projectionCellID" ippType="TLONG" ippName="" comment="projection cell identifier"/> -->
+    <!--  <map pspsName="stackVer" ippType="TSHORT" ippName="" comment="version number of this stack"/> -->
+    <!--  <map pspsName="obsTime" ippType="TDOUBLE" ippName="" comment=" Time of mid observation (unit = day)"/> -->
+    <!--  <map pspsName="xPos" ippType="TFLOAT" ippName="" comment="measured x on CCD from PSF fit"/> -->
+    <!--  <map pspsName="yPos" ippType="TFLOAT" ippName="" comment="measured y on CCD from PSF fit"/> -->
+    <!--  <map pspsName="xPosErr" ippType="TFLOAT" ippName="" comment="estimated error in x"/> -->
+    <!--  <map pspsName="yPosErr" ippType="TFLOAT" ippName="" comment="estimated error in y"/> -->
+          <map pspsName="instFlux" ippType="TFLOAT" ippName="PSF_INST_FLUX" comment="PSF instrumental flux"/> 
+          <map pspsName="instFluxErr" ippType="TFLOAT" ippName="PSF_INST_FLUX_SIG" comment="estimated error in instrumental flux"/> 
+    <!--  <map pspsName="peakFlux" ippType="TFLOAT" ippName="" comment="ratio of peak flux to total flux"/> -->
+          <map pspsName="sky" ippType="TFLOAT" ippName="SKY" comment="PSF sky level at source (adu)"/> 
+          <map pspsName="skyErr" ippType="TFLOAT" ippName="SKY_SIGMA" comment="estimated error in sky"/>
+          <map pspsName="sgSep" ippType="TFLOAT" ippName="EXT_NSIGMA" comment="star/galaxy separator"/>
+          <map pspsName="psfWidMajor" ippType="TFLOAT" ippName="PSF_MAJOR" comment="PSF width in major axis"/> 
+          <map pspsName="psfWidMinor" ippType="TFLOAT" ippName="PSF_MINOR" comment="PSF width in minor axis"/> 
+          <map pspsName="psfTheta" ippType="TFLOAT" ippName="PSF_THETA" comment="PSF orientation angle"/> 
+    <!--  <map pspsName="psfLikelihood" ippType="TFLOAT" ippName="" comment="PSF likelihood"/> -->
+          <map pspsName="psfCf" ippType="TFLOAT" ippName="PSF_QF" comment="PSF coverage factor"/>
+    <!--  <map pspsName="infoFlag" ippType="TLONG" ippName="" comment="indicator of strange propeties"/> -->
+    <!--  <map pspsName="nFrames" ippType="TLONG" ippName="" comment="number of frames contributing to source"/> -->
+    <!--  <map pspsName="wlSigma" ippType="TFLOAT" ippName="" comment="weak lensing sigma"/> -->
+    <!--  <map pspsName="eps1" ippType="TFLOAT" ippName="" comment="weak lensing vector element eps(1)"/> -->
+    <!--  <map pspsName="eps2" ippType="TFLOAT" ippName="" comment="weak lensing vector element eps(2)"/> -->
+    <!--  <map pspsName="Psm11" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sm(1,1)"/> -->
+    <!--  <map pspsName="Psm12" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sm(1,2)"/> -->
+    <!--  <map pspsName="Psm21" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sm(2,1)"/> -->
+    <!--  <map pspsName="Psm22" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sm(2,2)"/> -->
+    <!--  <map pspsName="Psh11" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sh(1,1)"/> -->
+    <!--  <map pspsName="Psh12" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sh(1,2)"/> -->
+    <!--  <map pspsName="Psh21" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sh(2,1)"/> -->
+    <!--  <map pspsName="Psh22" ippType="TFLOAT" ippName="" comment="weak lensing matrix element P_sh(2,2)"/> -->
+    <!--  <map pspsName="activeFlag" ippType="TBYTE" ippName="" comment="indicates whether this detection/orphan is still a detection/orphan"/> -->
+    <!--  <map pspsName="assocDate" ippType="TSTRING" ippName="" comment="date object association assigned"/> -->
+    <!--  <map pspsName="historyModNum" ippType="TSHORT" ippName="" comment="modification number in the O-D association history"/> -->
+    <!--  <map pspsName="dataRelease" ippType="TBYTE" ippName="" comment="Data release when this detection was originally taken."/> -->
+  </table>
+  <table name="StackApFlx" ippfitsextension="SkyChip.xsrc">
+    <!--  <map pspsName="objID" ippType="TLONGLONG" ippName="" comment="ODM object identifier"/> -->
+    <!--  <map pspsName="stackDetectID" ippType="TLONGLONG" ippName="" comment="ODM detection identifier"/> -->
+    <!--  <map pspsName="ippObjID" ippType="TLONGLONG" ippName="" comment="IPP object identifier"/> -->
+          <map pspsName="ippDetectID" ippType="TLONGLONG" ippName="IPP_IDET" comment="detection ID generated by IPP"/>
+    <!--  <map pspsName="filterID" ippType="TBYTE" ippName="" comment="filter identifier"/> -->
+    <!--  <map pspsName="stackTypeID" ippType="TBYTE" ippName="" comment="stack type identifier"/> -->
+    <!--   *** DONE IN CODE <map pspsName="surveyID" ippType="TBYTE" ippName="" comment="survey flag identifier"/> -->
+    <!--  <map pspsName="primaryF" ippType="TBYTE" ippName="" comment="identifies best stack detection for Stacks overlapping the same region of the sky."/> -->
+    <!--  <map pspsName="stackMetaID" ippType="TLONGLONG" ippName="" comment="stack identifier"/> -->
+    <!--  <map pspsName="isophotMag" ippType="TFLOAT" ippName="" comment="isophotal magnitude"/> -->
+    <!--  <map pspsName="isophotMagErr" ippType="TFLOAT" ippName="" comment="estimated error in isophotal magnitude"/> -->
+    <!--  <map pspsName="isophotMajAxis" ippType="TFLOAT" ippName="" comment="isophotal Major Axis"/> -->
+    <!--  <map pspsName="isophotMajAxisErr" ippType="TFLOAT" ippName="" comment="estimated error in isophotal Major Axis"/> -->
+    <!--  <map pspsName="isophotMinAxis" ippType="TFLOAT" ippName="" comment="isophotal Minor Axis"/> -->
+    <!--  <map pspsName="isophotMinAxisErr" ippType="TFLOAT" ippName="" comment="estimated error in isophotal Minor Axis"/> -->
+    <!--  <map pspsName="isophotMajAxisGrad" ippType="TFLOAT" ippName="" comment="isophotal major axis gradient"/> -->
+    <!--  <map pspsName="isophotMinAxisGrad" ippType="TFLOAT" ippName="" comment="isophotal minor axis gradient"/> -->
+    <!--  <map pspsName="isophotPA" ippType="TFLOAT" ippName="" comment="isophotal position angle"/> -->
+    <!--  <map pspsName="isophotPAErr" ippType="TFLOAT" ippName="" comment="estimated error in isophotal position angle"/> -->
+    <!--  <map pspsName="isophotPAGrad" ippType="TFLOAT" ippName="" comment="isophotal position angle gradient"/> -->
+          <map pspsName="petRadius" ippType="TFLOAT" ippName="PETRO_RADIUS" comment="Petrosian radius"/>
+          <map pspsName="petRadiusErr" ippType="TFLOAT" ippName="PETRO_RADIUS_ERR" comment="estimated error inPetrosian radius"/> 
+          <map pspsName="petMag" ippType="TFLOAT" ippName="PETRO_MAG" comment="Petrosian magntiude"/>
+          <map pspsName="petMagErr" ippType="TFLOAT" ippName="PETRO_MAG_ERR" comment="estimated error in petMag"/>
+          <map pspsName="petR50" ippType="TFLOAT" ippName="PETRO_RADIUS_50" comment="Petrosian radius at 50% light"/> 
+          <map pspsName="petR50Err" ippType="TFLOAT" ippName="PETRO_RADIUS_50" comment="estimated error inPetrosian radius at 50% light"/> 
+          <map pspsName="petR90" ippType="TFLOAT" ippName="PETRO_RADIUS_90" comment="Petrosian radius at 90% light"/>
+          <map pspsName="petR90Err" ippType="TFLOAT" ippName="PETRO_RADIUS_90_ERR" comment="estimated error in Petrosian radius at 90% light"/>
+    <!--  <map pspsName="petCf" ippType="TFLOAT" ippName="" comment="Petrosian fit coverage factor"/> -->
+    <!--  <map pspsName="flxR1" ippType="TFLOAT" ippName="" comment="Flux inside r = 1"/> -->
+    <!--  <map pspsName="flxR1Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR1"/> -->
+    <!--  <map pspsName="flxR1Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR1"/> -->
+    <!--  <map pspsName="flxR2" ippType="TFLOAT" ippName="" comment="Flux inside r = 2"/> -->
+    <!--  <map pspsName="flxR2Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR2"/> -->
+    <!--  <map pspsName="flxR2Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR2"/> -->
+    <!--  <map pspsName="flxR3" ippType="TFLOAT" ippName="" comment="Flux inside r = 3"/> -->
+    <!--  <map pspsName="flxR3Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR3"/> -->
+    <!--  <map pspsName="flxR3Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR3"/> -->
+    <!--  <map pspsName="flxR4" ippType="TFLOAT" ippName="" comment="Flux inside r = 4"/> -->
+    <!--  <map pspsName="flxR4Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR4"/> -->
+    <!--  <map pspsName="flxR4Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR4"/> -->
+    <!--  <map pspsName="flxR5" ippType="TFLOAT" ippName="" comment="Flux inside r = 5"/> -->
+    <!--  <map pspsName="flxR5Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR5"/> -->
+    <!--  <map pspsName="flxR5Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR5"/> -->
+    <!--  <map pspsName="flxR6" ippType="TFLOAT" ippName="" comment="Flux inside r = 6"/> -->
+    <!--  <map pspsName="flxR6Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR6"/> -->
+    <!--  <map pspsName="flxR6Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR6"/> -->
+    <!--  <map pspsName="flxR7" ippType="TFLOAT" ippName="" comment="Flux inside r = 7"/> -->
+    <!--  <map pspsName="flxR7Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR7"/> -->
+    <!--  <map pspsName="flxR7Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR7"/> -->
+    <!--  <map pspsName="flxR8" ippType="TFLOAT" ippName="" comment="Flux inside r = 8"/> -->
+    <!--  <map pspsName="flxR8Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR8"/> -->
+    <!--  <map pspsName="flxR8Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR8"/> -->
+    <!--  <map pspsName="flxR9" ippType="TFLOAT" ippName="" comment="Flux inside r = 9"/> -->
+    <!--  <map pspsName="flxR9Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR9"/> -->
+    <!--  <map pspsName="flxR9Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR9"/> -->
+    <!--  <map pspsName="flxR10" ippType="TFLOAT" ippName="" comment="Flux inside r = 10"/> -->
+    <!--  <map pspsName="flxR10Err" ippType="TFLOAT" ippName="" comment="estimated error is flxR10"/> -->
+    <!--  <map pspsName="flxR10Var" ippType="TFLOAT" ippName="" comment="estimated variance in flxR10"/> -->
+    <!--  <map pspsName="c1FlxR1" ippType="TFLOAT" ippName="" comment="Flux inside r = 1"/> -->
+    <!--  <map pspsName="c1FlxR1Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR1"/> -->
+    <!--  <map pspsName="c1FlxR1Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR1"/> -->
+    <!--  <map pspsName="c1FlxR2" ippType="TFLOAT" ippName="" comment="Flux inside r = 2"/> -->
+    <!--  <map pspsName="c1FlxR2Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR2"/> -->
+    <!--  <map pspsName="c1FlxR2Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR2"/> -->
+    <!--  <map pspsName="c1FlxR3" ippType="TFLOAT" ippName="" comment="Flux inside r = 3"/> -->
+    <!--  <map pspsName="c1FlxR3Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR3"/> -->
+    <!--  <map pspsName="c1FlxR3Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR3"/> -->
+    <!--  <map pspsName="c1FlxR4" ippType="TFLOAT" ippName="" comment="Flux inside r = 4"/> -->
+    <!--  <map pspsName="c1FlxR4Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR4"/> -->
+    <!--  <map pspsName="c1FlxR4Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR4"/> -->
+    <!--  <map pspsName="c1FlxR5" ippType="TFLOAT" ippName="" comment="Flux inside r = 5"/> -->
+    <!--  <map pspsName="c1FlxR5Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR5"/> -->
+    <!--  <map pspsName="c1FlxR5Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR5"/> -->
+    <!--  <map pspsName="c1FlxR6" ippType="TFLOAT" ippName="" comment="Flux inside r = 6"/> -->
+    <!--  <map pspsName="c1FlxR6Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR6"/> -->
+    <!--  <map pspsName="c1FlxR6Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR6"/> -->
+    <!--  <map pspsName="c1FlxR7" ippType="TFLOAT" ippName="" comment="Flux inside r = 7"/> -->
+    <!--  <map pspsName="c1FlxR7Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR7"/> -->
+    <!--  <map pspsName="c1FlxR7Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR7"/> -->
+    <!--  <map pspsName="c1FlxR8" ippType="TFLOAT" ippName="" comment="Flux inside r = 8"/> -->
+    <!--  <map pspsName="c1FlxR8Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR8"/> -->
+    <!--  <map pspsName="c1FlxR8Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR8"/> -->
+    <!--  <map pspsName="c1FlxR9" ippType="TFLOAT" ippName="" comment="Flux inside r = 9"/> -->
+    <!--  <map pspsName="c1FlxR9Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR9"/> -->
+    <!--  <map pspsName="c1FlxR9Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR9"/> -->
+    <!--  <map pspsName="c1FlxR10" ippType="TFLOAT" ippName="" comment="Flux inside r = 10"/> -->
+    <!--  <map pspsName="c1FlxR10Err" ippType="TFLOAT" ippName="" comment="estimated error is c1FlxR10"/> -->
+    <!--  <map pspsName="c1FlxR10Var" ippType="TFLOAT" ippName="" comment="estimated variance in c1FlxR10"/> -->
+    <!--  <map pspsName="c2FlxR1" ippType="TFLOAT" ippName="" comment="Flux inside r = 1"/> -->
+    <!--  <map pspsName="c2FlxR1Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR1"/> -->
+    <!--  <map pspsName="c2FlxR1Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR1"/> -->
+    <!--  <map pspsName="c2FlxR2" ippType="TFLOAT" ippName="" comment="Flux inside r = 2"/> -->
+    <!--  <map pspsName="c2FlxR2Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR2"/> -->
+    <!--  <map pspsName="c2FlxR2Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR2"/> -->
+    <!--  <map pspsName="c2FlxR3" ippType="TFLOAT" ippName="" comment="Flux inside r = 3"/> -->
+    <!--  <map pspsName="c2FlxR3Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR3"/> -->
+    <!--  <map pspsName="c2FlxR3Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR3"/> -->
+    <!--  <map pspsName="c2FlxR4" ippType="TFLOAT" ippName="" comment="Flux inside r = 4"/> -->
+    <!--  <map pspsName="c2FlxR4Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR4"/> -->
+    <!--  <map pspsName="c2FlxR4Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR4"/> -->
+    <!--  <map pspsName="c2FlxR5" ippType="TFLOAT" ippName="" comment="Flux inside r = 5"/> -->
+    <!--  <map pspsName="c2FlxR5Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR5"/> -->
+    <!--  <map pspsName="c2FlxR5Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR5"/> -->
+    <!--  <map pspsName="c2FlxR6" ippType="TFLOAT" ippName="" comment="Flux inside r = 6"/> -->
+    <!--  <map pspsName="c2FlxR6Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR6"/> -->
+    <!--  <map pspsName="c2FlxR6Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR6"/> -->
+    <!--  <map pspsName="c2FlxR7" ippType="TFLOAT" ippName="" comment="Flux inside r = 7"/> -->
+    <!--  <map pspsName="c2FlxR7Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR7"/> -->
+    <!--  <map pspsName="c2FlxR7Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR7"/> -->
+    <!--  <map pspsName="c2FlxR8" ippType="TFLOAT" ippName="" comment="Flux inside r = 8"/> -->
+    <!--  <map pspsName="c2FlxR8Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR8"/> -->
+    <!--  <map pspsName="c2FlxR8Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR8"/> -->
+    <!--  <map pspsName="c2FlxR9" ippType="TFLOAT" ippName="" comment="Flux inside r = 9"/> -->
+    <!--  <map pspsName="c2FlxR9Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR9"/> -->
+    <!--  <map pspsName="c2FlxR9Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR9"/> -->
+    <!--  <map pspsName="c2FlxR10" ippType="TFLOAT" ippName="" comment="Flux inside r = 10"/> -->
+    <!--  <map pspsName="c2FlxR10Err" ippType="TFLOAT" ippName="" comment="estimated error is c2FlxR10"/> -->
+    <!--  <map pspsName="c2FlxR10Var" ippType="TFLOAT" ippName="" comment="estimated variance in c2FlxR10"/> -->
+    <!--  <map pspsName="c3FlxR1" ippType="TFLOAT" ippName="" comment="Flux inside r = 1"/> -->
+    <!--  <map pspsName="c3FlxR1Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR1"/> -->
+    <!--  <map pspsName="c3FlxR1Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR1"/> -->
+    <!--  <map pspsName="c3FlxR2" ippType="TFLOAT" ippName="" comment="Flux inside r = 2"/> -->
+    <!--  <map pspsName="c3FlxR2Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR2"/> -->
+    <!--  <map pspsName="c3FlxR2Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR2"/> -->
+    <!--  <map pspsName="c3FlxR3" ippType="TFLOAT" ippName="" comment="Flux inside r = 3"/> -->
+    <!--  <map pspsName="c3FlxR3Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR3"/> -->
+    <!--  <map pspsName="c3FlxR3Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR3"/> -->
+    <!--  <map pspsName="c3FlxR4" ippType="TFLOAT" ippName="" comment="Flux inside r = 4"/> -->
+    <!--  <map pspsName="c3FlxR4Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR4"/> -->
+    <!--  <map pspsName="c3FlxR4Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR4"/> -->
+    <!--  <map pspsName="c3FlxR5" ippType="TFLOAT" ippName="" comment="Flux inside r = 5"/> -->
+    <!--  <map pspsName="c3FlxR5Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR5"/> -->
+    <!--  <map pspsName="c3FlxR5Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR5"/> -->
+    <!--  <map pspsName="c3FlxR6" ippType="TFLOAT" ippName="" comment="Flux inside r = 6"/> -->
+    <!--  <map pspsName="c3FlxR6Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR6"/> -->
+    <!--  <map pspsName="c3FlxR6Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR6"/> -->
+    <!--  <map pspsName="c3FlxR7" ippType="TFLOAT" ippName="" comment="Flux inside r = 7"/> -->
+    <!--  <map pspsName="c3FlxR7Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR7"/> -->
+    <!--  <map pspsName="c3FlxR7Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR7"/> -->
+    <!--  <map pspsName="c3FlxR8" ippType="TFLOAT" ippName="" comment="Flux inside r = 8"/> -->
+    <!--  <map pspsName="c3FlxR8Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR8"/> -->
+    <!--  <map pspsName="c3FlxR8Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR8"/> -->
+    <!--  <map pspsName="c3FlxR9" ippType="TFLOAT" ippName="" comment="Flux inside r = 9"/> -->
+    <!--  <map pspsName="c3FlxR9Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR9"/> -->
+    <!--  <map pspsName="c3FlxR9Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR9"/> -->
+    <!--  <map pspsName="c3FlxR10" ippType="TFLOAT" ippName="" comment="Flux inside r = 10"/> -->
+    <!--  <map pspsName="c3FlxR10Err" ippType="TFLOAT" ippName="" comment="estimated error is c3FlxR10"/> -->
+    <!--  <map pspsName="c3FlxR10Var" ippType="TFLOAT" ippName="" comment="estimated variance in c3FlxR10"/> -->
+    <!--  <map pspsName="logC" ippType="TFLOAT" ippName="" comment="Abraham concentration index"/> -->
+    <!--  <map pspsName="logA" ippType="TFLOAT" ippName="" comment="Abraham asymmetry index"/> -->
+    <!--  <map pspsName="activeFlag" ippType="TBYTE" ippName="" comment="indicates whether this detection/orphan is still a detection/orphan"/> -->
+    <!--  <map pspsName="dataRelease" ippType="TBYTE" ippName="" comment="Data release when this detection was taken"/> -->
+  </table>
+  <table name="StackModelFit" ippfitsextension="">
+    <!--  <map pspsName="objID" ippType="TLONGLONG" ippName="" comment="ODM object identifier"/> -->
+    <!--  <map pspsName="stackDetectID" ippType="TLONGLONG" ippName="" comment="ODM detection identifier"/> -->
+    <!--  <map pspsName="ippObjID" ippType="TLONGLONG" ippName="" comment="IPP object identifier"/> -->
+    <!--  <map pspsName="ippDetectID" ippType="TLONGLONG" ippName="" comment="detection ID generated by IPP"/> -->
+    <!--  <map pspsName="filterID" ippType="TBYTE" ippName="" comment="filter identifier"/> -->
+    <!--  <map pspsName="stackTypeID" ippType="TBYTE" ippName="" comment="stack type identifier"/> -->
+    <!--  <map pspsName="surveyID" ippType="TBYTE" ippName="" comment="survey flag identifier"/> -->
+    <!--  <map pspsName="primaryF" ippType="TBYTE" ippName="" comment="identifies best stack detection for Stacks overlapping the same region of the sky."/> -->
+    <!--  <map pspsName="stackMetaID" ippType="TLONGLONG" ippName="" comment="stack identifier"/> -->
+    <!--  <map pspsName="deVRadius" ippType="TFLOAT" ippName="" comment="deVaucouleurs radius"/> -->
+    <!--  <map pspsName="deVRadiusErr" ippType="TFLOAT" ippName="" comment="estimated error in deVaucouleurs radius"/> -->
+    <!--  <map pspsName="deVMag" ippType="TFLOAT" ippName="" comment="deVaucouleurs magntiude"/> -->
+    <!--  <map pspsName="deVMagErr" ippType="TFLOAT" ippName="" comment="estimated error in deV_mag"/> -->
+    <!--  <map pspsName="deVAb" ippType="TFLOAT" ippName="" comment="deVaucoulerus axis ratio"/> -->
+    <!--  <map pspsName="deVAbErr" ippType="TFLOAT" ippName="" comment="estimated error in deVaucoulerus axis ratio"/> -->
+    <!--  <map pspsName="raDeVOff" ippType="TFLOAT" ippName="" comment="Offset in RA of deVaucouleurs fit from PSF RA"/> -->
+    <!--  <map pspsName="decDeVOff" ippType="TFLOAT" ippName="" comment="Offset in DEC of deVaucouleurs fit from PSF DEC"/> -->
+    <!--  <map pspsName="raDeVOffErr" ippType="TFLOAT" ippName="" comment="estimated error in ra offset"/> -->
+    <!--  <map pspsName="decDeVOffErr" ippType="TFLOAT" ippName="" comment="estimated error in dec offset"/> -->
+    <!--  <map pspsName="deVCf" ippType="TFLOAT" ippName="" comment="deVaucouleurs fit coverage factor"/> -->
+    <!--  <map pspsName="deVLikelihood" ippType="TFLOAT" ippName="" comment="deVaucouleurs fit likelihood factor"/> -->
+    <!--  <map pspsName="deVCovar11" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar12" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar13" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar14" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar15" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar16" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar22" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar23" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar24" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar25" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar26" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar33" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar34" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar35" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar36" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar44" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar45" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar46" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar55" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar56" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="deVCovar66" ippType="TFLOAT" ippName="" comment="covariance for deVaucouleurs fit"/> -->
+    <!--  <map pspsName="expRadius" ippType="TFLOAT" ippName="" comment="Exponential fit radius"/> -->
+    <!--  <map pspsName="expRadiusErr" ippType="TFLOAT" ippName="" comment="estimated error in Exponential fit radius"/> -->
+    <!--  <map pspsName="expMag" ippType="TFLOAT" ippName="" comment="Exponential fit magntiude"/> -->
+    <!--  <map pspsName="expMagErr" ippType="TFLOAT" ippName="" comment="estimated error in expMag"/> -->
+    <!--  <map pspsName="expAb" ippType="TFLOAT" ippName="" comment="Exponential fit axis ratio"/> -->
+    <!--  <map pspsName="expAbErr" ippType="TFLOAT" ippName="" comment="estimated error in Exponential fit axis ratio"/> -->
+    <!--  <map pspsName="raExpOff" ippType="TFLOAT" ippName="" comment="Offset in RA of Exponential fit from PSF RA"/> -->
+    <!--  <map pspsName="decExpOff" ippType="TFLOAT" ippName="" comment="Offset in DEC of Exponential fit from PSF DEC"/> -->
+    <!--  <map pspsName="raExpOffErr" ippType="TFLOAT" ippName="" comment="estimated error in raExpOff"/> -->
+    <!--  <map pspsName="decExpOffErr" ippType="TFLOAT" ippName="" comment="estimated error in decExpOff"/> -->
+    <!--  <map pspsName="expCf" ippType="TFLOAT" ippName="" comment="Exponential fit coverage factor"/> -->
+    <!--  <map pspsName="expLikelihood" ippType="TFLOAT" ippName="" comment="Exponential fit likelihood factor"/> -->
+    <!--  <map pspsName="expCovar11" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar12" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar13" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar14" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar15" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar16" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar22" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar23" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar24" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar25" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar26" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar33" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar34" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar35" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar36" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar44" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar45" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar46" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar55" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar56" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="expCovar66" ippType="TFLOAT" ippName="" comment="covariance for Exponential fit"/> -->
+    <!--  <map pspsName="serRadius" ippType="TFLOAT" ippName="" comment="Sersic radius"/> -->
+    <!--  <map pspsName="serRadiusErr" ippType="TFLOAT" ippName="" comment="estimated error in Sersic radius"/> -->
+    <!--  <map pspsName="serMag" ippType="TFLOAT" ippName="" comment="Sersic magntiude"/> -->
+    <!--  <map pspsName="serMagErr" ippType="TFLOAT" ippName="" comment="estimated error in serMag"/> -->
+    <!--  <map pspsName="serAb" ippType="TFLOAT" ippName="" comment="Sersic axis ratio"/> -->
+    <!--  <map pspsName="serAbErr" ippType="TFLOAT" ippName="" comment="estimated error in Sersic axis ratio"/> -->
+    <!--  <map pspsName="serNu" ippType="TFLOAT" ippName="" comment="Sersic index"/> -->
+    <!--  <map pspsName="serNuErr" ippType="TFLOAT" ippName="" comment="estimated error in Sersic index"/> -->
+    <!--  <map pspsName="raSerOff" ippType="TFLOAT" ippName="" comment="Offset in RA of Sersic fit from PSF RA"/> -->
+    <!--  <map pspsName="decSerOff" ippType="TFLOAT" ippName="" comment="Offset in DEC of Sersic fit from PSF DEC"/> -->
+    <!--  <map pspsName="raSerOffErr" ippType="TFLOAT" ippName="" comment="estimated error in raSerOff"/> -->
+    <!--  <map pspsName="decSerOffErr" ippType="TFLOAT" ippName="" comment="estimated error in decSerOff"/> -->
+    <!--  <map pspsName="serCf" ippType="TFLOAT" ippName="" comment="Sersic fit coverage factor"/> -->
+    <!--  <map pspsName="serLikelihood" ippType="TFLOAT" ippName="" comment="Sersic fit likelihood factor"/> -->
+    <!--  <map pspsName="sersicCovar11" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar12" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar13" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar14" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar15" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar16" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar17" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar22" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar23" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar24" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar25" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar26" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar27" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar33" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar34" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar35" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar36" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar37" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar44" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar45" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar46" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar47" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar55" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar56" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar57" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar66" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar67" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="sersicCovar77" ippType="TFLOAT" ippName="" comment="covariance for Sersic fit"/> -->
+    <!--  <map pspsName="activeFlag" ippType="TBYTE" ippName="" comment="indicates whether this detection/orphan is still a detection/orphan"/> -->
+    <!--  <map pspsName="dataRelease" ippType="TBYTE" ippName="" comment="Data release when this detection was taken"/> -->
+  </table>
+  <table name="StackToImage" ippfitsextension="">
+    <!--  <map pspsName="stackMetaID" ippType="TLONGLONG" ippName="" comment="stack identifier"/> -->
+    <!--  <map pspsName="imageID" ippType="TLONGLONG" ippName="" comment="hashed exposure-ccdID identifier"/> -->
+  </table>
+</tabledata>
Index: /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/tables.xml
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/tables.xml	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/config/stack/tables.xml	(revision 29515)
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<tableDescriptions type="ST">
+<tableDescriptions type="stack">
   <table name="StackMeta">
     <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
@@ -30,27 +30,29 @@
     <column name="ctype1" type="TSTRING" default=" " comment="name of astrometric projection in RA"></column>
     <column name="ctype2" type="TSTRING" default=" " comment="name of astrometric projection in DEC"></column>
-    <column name="crval1" type="TFLOAT" default="-999" comment="RA corresponding to reference pixel"></column>
-    <column name="crval2" type="TFLOAT" default="-999" comment="DEC corresponding to reference pixel"></column>
-    <column name="crpix1" type="TFLOAT" default="-999" comment="reference pixel value for RA"></column>
-    <column name="crpix2" type="TFLOAT" default="-999" comment="reference pixel value for DEC"></column>
-    <column name="pc001001" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc001002" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002001" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002002" type="TFLOAT" default="-999" comment="elements of rotation/Dcale matrix"></column>
+    <column name="crval1" type="TDOUBLE" default="-999" comment="RA corresponding to reference pixel"></column>
+    <column name="crval2" type="TDOUBLE" default="-999" comment="DEC corresponding to reference pixel"></column>
+    <column name="crpix1" type="TDOUBLE" default="-999" comment="reference pixel value for RA"></column>
+    <column name="crpix2" type="TDOUBLE" default="-999" comment="reference pixel value for DEC"></column>
+    <column name="cdelt1" type="TDOUBLE" default="-999" comment="scale factor for RA"></column>
+    <column name="cdelt2" type="TDOUBLE" default="-999" comment="scale factor for DEC"></column>
+    <column name="pc001001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
+    <column name="pc001002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
+    <column name="pc002001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
+    <column name="pc002002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
     <column name="polyOrder" type="TBYTE" default="255" comment="polynomial order of astrometry fit"></column>
-    <column name="pca1x3y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y3" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x3y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y3" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y0" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y1" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y2" type="TFLOAT" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca1x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
+    <column name="pca2x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
     <column name="calibModNum" type="TSHORT" default="0" comment="calibration modification number"></column>
     <column name="dataRelease" type="TBYTE" default="0" comment="Data release"></column>
@@ -69,5 +71,5 @@
     <column name="projectionCellID" type="TLONG" default="-999" comment="projection cell identifier"></column>
     <column name="stackVer" type="TSHORT" default="-999" comment="version number of this stack"></column>
-    <column name="obsTime" type="TFLOAT" default="-999" comment=" Time of mid observation (unit = day)"></column>
+    <column name="obsTime" type="TDOUBLE" default="-999" comment=" Time of mid observation (unit = day)"></column>
     <column name="xPos" type="TFLOAT" default="-999" comment="measured x on CCD from PSF fit"></column>
     <column name="yPos" type="TFLOAT" default="-999" comment="measured y on CCD from PSF fit"></column>
@@ -103,62 +105,283 @@
     <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was originally taken."></column>
   </table>
-  <table name="SkinnyObject">
-    <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier index"></column>
-    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object number"></column>
-    <column name="projectionCellID" type="TLONG" default="-999" comment="projection cell identifier at discovery time"></column>
-    <column name="dataRelease" type="TBYTE" default="0" comment="Data release to propagate to the object"></column>
-  </table>
-  <table name="StackOrphan">
-    <column name="partitionKey" type="TLONGLONG" default="0" comment="partitioning key"></column>
-    <column name="stackDetectID" type="TLONGLONG" default="0" comment="detection identifier"></column>
+  <table name="StackApFlx">
+    <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier"></column>
+    <column name="stackDetectID" type="TLONGLONG" default="0" comment="ODM detection identifier"></column>
+    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object identifier"></column>
     <column name="ippDetectID" type="TLONGLONG" default="0" comment="detection ID generated by IPP"></column>
     <column name="filterID" type="TBYTE" default="0" comment="filter identifier"></column>
     <column name="stackTypeID" type="TBYTE" default="0" comment="stack type identifier"></column>
-    <column name="surveyID" type="TBYTE" default="0" comment="survey identifier"></column>
+    <column name="surveyID" type="TBYTE" default="0" comment="survey flag identifier"></column>
     <column name="primaryF" type="TBYTE" default="255" comment="identifies best stack detection for Stacks overlapping the same region of the sky."></column>
     <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
-    <column name="skyCellID" type="TLONG" default="-999" comment="skycell identifier"></column>
-    <column name="stackVer" type="TSHORT" default="-999" comment="version number of this stack"></column>
-    <column name="xPos" type="TFLOAT" default="-999" comment="measured x on CCD from PSF fit"></column>
-    <column name="yPos" type="TFLOAT" default="-999" comment="measured y on CCD from PSF fit"></column>
-    <column name="xPosErr" type="TFLOAT" default="-999" comment="estimated error in x"></column>
-    <column name="yPosErr" type="TFLOAT" default="-999" comment="estimated error in y"></column>
-    <column name="instFlux" type="TFLOAT" default="-999" comment="PSF instrumental flux"></column>
-    <column name="instFluxErr" type="TFLOAT" default="-999" comment="estimated error in instrumental flux"></column>
-    <column name="peakFlux" type="TFLOAT" default="-999" comment="ratio of peak flux to total flux"></column>
-    <column name="sky" type="TFLOAT" default="-999" comment="PSF sky level at source (adu)"></column>
-    <column name="skyErr" type="TFLOAT" default="-999" comment="estimated error in sky"></column>
-    <column name="sgSep" type="TFLOAT" default="-999" comment="star/galaxy separator"></column>
-    <column name="psfWidMajor" type="TFLOAT" default="-999" comment="PSF width in major axis"></column>
-    <column name="psfWidMinor" type="TFLOAT" default="-999" comment="PSF width in minor axis"></column>
-    <column name="psfTheta" type="TFLOAT" default="-999" comment="PSF orientation angle"></column>
-    <column name="psfLikelihood" type="TFLOAT" default="-999" comment="PSF likelihood"></column>
-    <column name="psfCf" type="TFLOAT" default="-999" comment="PSF coverage factor"></column>
-    <column name="infoFlag" type="TLONG" default="-999" comment="indicator of strange propeties"></column>
-    <column name="nFrames" type="TLONG" default="-999" comment="number of frames contributing to source"></column>
-    <column name="wlSigma" type="TFLOAT" default="-999" comment="weak lensing sigma"></column>
-    <column name="eps1" type="TFLOAT" default="-999" comment="weak lensing vector element eps(1)"></column>
-    <column name="eps2" type="TFLOAT" default="-999" comment="weak lensing vector element eps(2)"></column>
-    <column name="Psm11" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(1,1)"></column>
-    <column name="Psm12" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(1,2)"></column>
-    <column name="Psm21" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(2,1)"></column>
-    <column name="Psm22" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sm(2,2)"></column>
-    <column name="Psh11" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(1,1)"></column>
-    <column name="Psh12" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(1,2)"></column>
-    <column name="Psh21" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(2,1)"></column>
-    <column name="Psh22" type="TFLOAT" default="-999" comment="weak lensing matrix element P_sh(2,2)"></column>
+    <column name="isophotMag" type="TFLOAT" default="-999" comment="isophotal magnitude"></column>
+    <column name="isophotMagErr" type="TFLOAT" default="-999" comment="estimated error in isophotal magnitude"></column>
+    <column name="isophotMajAxis" type="TFLOAT" default="-999" comment="isophotal Major Axis"></column>
+    <column name="isophotMajAxisErr" type="TFLOAT" default="-999" comment="estimated error in isophotal Major Axis"></column>
+    <column name="isophotMinAxis" type="TFLOAT" default="-999" comment="isophotal Minor Axis"></column>
+    <column name="isophotMinAxisErr" type="TFLOAT" default="-999" comment="estimated error in isophotal Minor Axis"></column>
+    <column name="isophotMajAxisGrad" type="TFLOAT" default="-999" comment="isophotal major axis gradient"></column>
+    <column name="isophotMinAxisGrad" type="TFLOAT" default="-999" comment="isophotal minor axis gradient"></column>
+    <column name="isophotPA" type="TFLOAT" default="-999" comment="isophotal position angle"></column>
+    <column name="isophotPAErr" type="TFLOAT" default="-999" comment="estimated error in isophotal position angle"></column>
+    <column name="isophotPAGrad" type="TFLOAT" default="-999" comment="isophotal position angle gradient"></column>
+    <column name="petRadius" type="TFLOAT" default="-999" comment="Petrosian radius"></column>
+    <column name="petRadiusErr" type="TFLOAT" default="-999" comment="estimated error inPetrosian radius"></column>
+    <column name="petMag" type="TFLOAT" default="-999" comment="Petrosian magntiude"></column>
+    <column name="petMagErr" type="TFLOAT" default="-999" comment="estimated error in petMag"></column>
+    <column name="petR50" type="TFLOAT" default="-999" comment="Petrosian radius at 50% light"></column>
+    <column name="petR50Err" type="TFLOAT" default="-999" comment="estimated error inPetrosian radius at 50% light"></column>
+    <column name="petR90" type="TFLOAT" default="-999" comment="Petrosian radius at 90% light"></column>
+    <column name="petR90Err" type="TFLOAT" default="-999" comment="estimated error in Petrosian radius at 90% light"></column>
+    <column name="petCf" type="TFLOAT" default="-999" comment="Petrosian fit coverage factor"></column>
+    <column name="flxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
+    <column name="flxR1Err" type="TFLOAT" default="-999" comment="estimated error is flxR1"></column>
+    <column name="flxR1Var" type="TFLOAT" default="-999" comment="estimated variance in flxR1"></column>
+    <column name="flxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
+    <column name="flxR2Err" type="TFLOAT" default="-999" comment="estimated error is flxR2"></column>
+    <column name="flxR2Var" type="TFLOAT" default="-999" comment="estimated variance in flxR2"></column>
+    <column name="flxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
+    <column name="flxR3Err" type="TFLOAT" default="-999" comment="estimated error is flxR3"></column>
+    <column name="flxR3Var" type="TFLOAT" default="-999" comment="estimated variance in flxR3"></column>
+    <column name="flxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
+    <column name="flxR4Err" type="TFLOAT" default="-999" comment="estimated error is flxR4"></column>
+    <column name="flxR4Var" type="TFLOAT" default="-999" comment="estimated variance in flxR4"></column>
+    <column name="flxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
+    <column name="flxR5Err" type="TFLOAT" default="-999" comment="estimated error is flxR5"></column>
+    <column name="flxR5Var" type="TFLOAT" default="-999" comment="estimated variance in flxR5"></column>
+    <column name="flxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
+    <column name="flxR6Err" type="TFLOAT" default="-999" comment="estimated error is flxR6"></column>
+    <column name="flxR6Var" type="TFLOAT" default="-999" comment="estimated variance in flxR6"></column>
+    <column name="flxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
+    <column name="flxR7Err" type="TFLOAT" default="-999" comment="estimated error is flxR7"></column>
+    <column name="flxR7Var" type="TFLOAT" default="-999" comment="estimated variance in flxR7"></column>
+    <column name="flxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
+    <column name="flxR8Err" type="TFLOAT" default="-999" comment="estimated error is flxR8"></column>
+    <column name="flxR8Var" type="TFLOAT" default="-999" comment="estimated variance in flxR8"></column>
+    <column name="flxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
+    <column name="flxR9Err" type="TFLOAT" default="-999" comment="estimated error is flxR9"></column>
+    <column name="flxR9Var" type="TFLOAT" default="-999" comment="estimated variance in flxR9"></column>
+    <column name="flxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
+    <column name="flxR10Err" type="TFLOAT" default="-999" comment="estimated error is flxR10"></column>
+    <column name="flxR10Var" type="TFLOAT" default="-999" comment="estimated variance in flxR10"></column>
+    <column name="c1FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
+    <column name="c1FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR1"></column>
+    <column name="c1FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR1"></column>
+    <column name="c1FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
+    <column name="c1FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR2"></column>
+    <column name="c1FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR2"></column>
+    <column name="c1FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
+    <column name="c1FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR3"></column>
+    <column name="c1FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR3"></column>
+    <column name="c1FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
+    <column name="c1FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR4"></column>
+    <column name="c1FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR4"></column>
+    <column name="c1FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
+    <column name="c1FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR5"></column>
+    <column name="c1FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR5"></column>
+    <column name="c1FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
+    <column name="c1FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR6"></column>
+    <column name="c1FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR6"></column>
+    <column name="c1FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
+    <column name="c1FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR7"></column>
+    <column name="c1FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR7"></column>
+    <column name="c1FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
+    <column name="c1FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR8"></column>
+    <column name="c1FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR8"></column>
+    <column name="c1FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
+    <column name="c1FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR9"></column>
+    <column name="c1FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR9"></column>
+    <column name="c1FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
+    <column name="c1FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c1FlxR10"></column>
+    <column name="c1FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c1FlxR10"></column>
+    <column name="c2FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
+    <column name="c2FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR1"></column>
+    <column name="c2FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR1"></column>
+    <column name="c2FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
+    <column name="c2FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR2"></column>
+    <column name="c2FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR2"></column>
+    <column name="c2FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
+    <column name="c2FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR3"></column>
+    <column name="c2FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR3"></column>
+    <column name="c2FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
+    <column name="c2FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR4"></column>
+    <column name="c2FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR4"></column>
+    <column name="c2FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
+    <column name="c2FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR5"></column>
+    <column name="c2FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR5"></column>
+    <column name="c2FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
+    <column name="c2FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR6"></column>
+    <column name="c2FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR6"></column>
+    <column name="c2FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
+    <column name="c2FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR7"></column>
+    <column name="c2FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR7"></column>
+    <column name="c2FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
+    <column name="c2FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR8"></column>
+    <column name="c2FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR8"></column>
+    <column name="c2FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
+    <column name="c2FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR9"></column>
+    <column name="c2FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR9"></column>
+    <column name="c2FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
+    <column name="c2FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c2FlxR10"></column>
+    <column name="c2FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c2FlxR10"></column>
+    <column name="c3FlxR1" type="TFLOAT" default="-999" comment="Flux inside r = 1"></column>
+    <column name="c3FlxR1Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR1"></column>
+    <column name="c3FlxR1Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR1"></column>
+    <column name="c3FlxR2" type="TFLOAT" default="-999" comment="Flux inside r = 2"></column>
+    <column name="c3FlxR2Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR2"></column>
+    <column name="c3FlxR2Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR2"></column>
+    <column name="c3FlxR3" type="TFLOAT" default="-999" comment="Flux inside r = 3"></column>
+    <column name="c3FlxR3Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR3"></column>
+    <column name="c3FlxR3Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR3"></column>
+    <column name="c3FlxR4" type="TFLOAT" default="-999" comment="Flux inside r = 4"></column>
+    <column name="c3FlxR4Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR4"></column>
+    <column name="c3FlxR4Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR4"></column>
+    <column name="c3FlxR5" type="TFLOAT" default="-999" comment="Flux inside r = 5"></column>
+    <column name="c3FlxR5Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR5"></column>
+    <column name="c3FlxR5Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR5"></column>
+    <column name="c3FlxR6" type="TFLOAT" default="-999" comment="Flux inside r = 6"></column>
+    <column name="c3FlxR6Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR6"></column>
+    <column name="c3FlxR6Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR6"></column>
+    <column name="c3FlxR7" type="TFLOAT" default="-999" comment="Flux inside r = 7"></column>
+    <column name="c3FlxR7Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR7"></column>
+    <column name="c3FlxR7Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR7"></column>
+    <column name="c3FlxR8" type="TFLOAT" default="-999" comment="Flux inside r = 8"></column>
+    <column name="c3FlxR8Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR8"></column>
+    <column name="c3FlxR8Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR8"></column>
+    <column name="c3FlxR9" type="TFLOAT" default="-999" comment="Flux inside r = 9"></column>
+    <column name="c3FlxR9Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR9"></column>
+    <column name="c3FlxR9Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR9"></column>
+    <column name="c3FlxR10" type="TFLOAT" default="-999" comment="Flux inside r = 10"></column>
+    <column name="c3FlxR10Err" type="TFLOAT" default="-999" comment="estimated error is c3FlxR10"></column>
+    <column name="c3FlxR10Var" type="TFLOAT" default="-999" comment="estimated variance in c3FlxR10"></column>
+    <column name="logC" type="TFLOAT" default="-999" comment="Abraham concentration index"></column>
+    <column name="logA" type="TFLOAT" default="-999" comment="Abraham asymmetry index"></column>
     <column name="activeFlag" type="TBYTE" default="-999" comment="indicates whether this detection/orphan is still a detection/orphan"></column>
-    <column name="assocDate" type="TSTRING" default="28881231" comment="date object association assigned"></column>
-    <column name="historyModNum" type="TSHORT" default="0" comment="modification number in the O-D association history"></column>
-    <column name="dataRelease" type="TBYTE" default="0" comment="Data release to reproduce queries through data releases"></column>
-  </table>
-  <table name="ObjectCalColor">
+    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was taken"></column>
+  </table>
+  <table name="StackModelFit">
     <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier"></column>
-    <column name="ippObjID" type="TLONGLONG" default="0" comment="ipp object identifier"></column>
+    <column name="stackDetectID" type="TLONGLONG" default="0" comment="ODM detection identifier"></column>
+    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object identifier"></column>
+    <column name="ippDetectID" type="TLONGLONG" default="0" comment="detection ID generated by IPP"></column>
     <column name="filterID" type="TBYTE" default="0" comment="filter identifier"></column>
-    <column name="calColor" type="TFLOAT" default="0" comment=" color adopted for magnitude calculation (unit = mag)"></column>
-    <column name="calColorErr" type="TFLOAT" default="0" comment=" error in calibrating color (unit = mag)"></column>
-    <column name="calibModNum" type="TSHORT" default="0" comment="calibration modification number"></column>
-    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this color calibration was established"></column>
+    <column name="stackTypeID" type="TBYTE" default="0" comment="stack type identifier"></column>
+    <column name="surveyID" type="TBYTE" default="0" comment="survey flag identifier"></column>
+    <column name="primaryF" type="TBYTE" default="255" comment="identifies best stack detection for Stacks overlapping the same region of the sky."></column>
+    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
+    <column name="deVRadius" type="TFLOAT" default="-999" comment="deVaucouleurs radius"></column>
+    <column name="deVRadiusErr" type="TFLOAT" default="-999" comment="estimated error in deVaucouleurs radius"></column>
+    <column name="deVMag" type="TFLOAT" default="-999" comment="deVaucouleurs magntiude"></column>
+    <column name="deVMagErr" type="TFLOAT" default="-999" comment="estimated error in deV_mag"></column>
+    <column name="deVAb" type="TFLOAT" default="-999" comment="deVaucoulerus axis ratio"></column>
+    <column name="deVAbErr" type="TFLOAT" default="-999" comment="estimated error in deVaucoulerus axis ratio"></column>
+    <column name="raDeVOff" type="TFLOAT" default="-999" comment="Offset in RA of deVaucouleurs fit from PSF RA"></column>
+    <column name="decDeVOff" type="TFLOAT" default="-999" comment="Offset in DEC of deVaucouleurs fit from PSF DEC"></column>
+    <column name="raDeVOffErr" type="TFLOAT" default="-999" comment="estimated error in ra offset"></column>
+    <column name="decDeVOffErr" type="TFLOAT" default="-999" comment="estimated error in dec offset"></column>
+    <column name="deVCf" type="TFLOAT" default="-999" comment="deVaucouleurs fit coverage factor"></column>
+    <column name="deVLikelihood" type="TFLOAT" default="-999" comment="deVaucouleurs fit likelihood factor"></column>
+    <column name="deVCovar11" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar12" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar13" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar14" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar15" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar16" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar22" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar23" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar24" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar25" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar26" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar33" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar34" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar35" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar36" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar44" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar45" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar46" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar55" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar56" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="deVCovar66" type="TFLOAT" default="-999" comment="covariance for deVaucouleurs fit"></column>
+    <column name="expRadius" type="TFLOAT" default="-999" comment="Exponential fit radius"></column>
+    <column name="expRadiusErr" type="TFLOAT" default="-999" comment="estimated error in Exponential fit radius"></column>
+    <column name="expMag" type="TFLOAT" default="-999" comment="Exponential fit magntiude"></column>
+    <column name="expMagErr" type="TFLOAT" default="-999" comment="estimated error in expMag"></column>
+    <column name="expAb" type="TFLOAT" default="-999" comment="Exponential fit axis ratio"></column>
+    <column name="expAbErr" type="TFLOAT" default="-999" comment="estimated error in Exponential fit axis ratio"></column>
+    <column name="raExpOff" type="TFLOAT" default="-999" comment="Offset in RA of Exponential fit from PSF RA"></column>
+    <column name="decExpOff" type="TFLOAT" default="-999" comment="Offset in DEC of Exponential fit from PSF DEC"></column>
+    <column name="raExpOffErr" type="TFLOAT" default="-999" comment="estimated error in raExpOff"></column>
+    <column name="decExpOffErr" type="TFLOAT" default="-999" comment="estimated error in decExpOff"></column>
+    <column name="expCf" type="TFLOAT" default="-999" comment="Exponential fit coverage factor"></column>
+    <column name="expLikelihood" type="TFLOAT" default="-999" comment="Exponential fit likelihood factor"></column>
+    <column name="expCovar11" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar12" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar13" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar14" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar15" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar16" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar22" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar23" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar24" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar25" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar26" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar33" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar34" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar35" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar36" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar44" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar45" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar46" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar55" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar56" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="expCovar66" type="TFLOAT" default="-999" comment="covariance for Exponential fit"></column>
+    <column name="serRadius" type="TFLOAT" default="-999" comment="Sersic radius"></column>
+    <column name="serRadiusErr" type="TFLOAT" default="-999" comment="estimated error in Sersic radius"></column>
+    <column name="serMag" type="TFLOAT" default="-999" comment="Sersic magntiude"></column>
+    <column name="serMagErr" type="TFLOAT" default="-999" comment="estimated error in serMag"></column>
+    <column name="serAb" type="TFLOAT" default="-999" comment="Sersic axis ratio"></column>
+    <column name="serAbErr" type="TFLOAT" default="-999" comment="estimated error in Sersic axis ratio"></column>
+    <column name="serNu" type="TFLOAT" default="-999" comment="Sersic index"></column>
+    <column name="serNuErr" type="TFLOAT" default="-999" comment="estimated error in Sersic index"></column>
+    <column name="raSerOff" type="TFLOAT" default="-999" comment="Offset in RA of Sersic fit from PSF RA"></column>
+    <column name="decSerOff" type="TFLOAT" default="-999" comment="Offset in DEC of Sersic fit from PSF DEC"></column>
+    <column name="raSerOffErr" type="TFLOAT" default="-999" comment="estimated error in raSerOff"></column>
+    <column name="decSerOffErr" type="TFLOAT" default="-999" comment="estimated error in decSerOff"></column>
+    <column name="serCf" type="TFLOAT" default="-999" comment="Sersic fit coverage factor"></column>
+    <column name="serLikelihood" type="TFLOAT" default="-999" comment="Sersic fit likelihood factor"></column>
+    <column name="sersicCovar11" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar12" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar13" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar14" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar15" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar16" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar17" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar22" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar23" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar24" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar25" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar26" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar27" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar33" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar34" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar35" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar36" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar37" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar44" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar45" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar46" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar47" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar55" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar56" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar57" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar66" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar67" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="sersicCovar77" type="TFLOAT" default="-999" comment="covariance for Sersic fit"></column>
+    <column name="activeFlag" type="TBYTE" default="-999" comment="indicates whether this detection/orphan is still a detection/orphan"></column>
+    <column name="dataRelease" type="TBYTE" default="0" comment="Data release when this detection was taken"></column>
+  </table>
+  <table name="StackToImage">
+    <column name="stackMetaID" type="TLONGLONG" default="0" comment="stack identifier"></column>
+    <column name="imageID" type="TLONGLONG" default="0" comment="hashed exposure-ccdID identifier"></column>
   </table>
 </tableDescriptions>
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl	(revision 29515)
@@ -10,5 +10,5 @@
 use ippToPsps::IppToPspsDb;
 use ippToPsps::Datastore;
-use ippToPsps::BatchManager;
+use ippToPsps::Batch;
 
 my $singleBatch = undef;
@@ -19,4 +19,5 @@
 my $product = undef;
 my $filePath = undef;
+my $removeUnprocessed = undef;
 
 
@@ -27,4 +28,5 @@
         'product|p=s' => \$product,
         'location|l=s' => \$filePath,
+        'remove|r' => \$removeUnprocessed,
         'verbose|v' => \$verbose,
         'save_temps|s' => \$save_temps
@@ -32,27 +34,31 @@
 
 if (!defined $product) {
-    print "* OPTIONAL: a datastore product name        -p <name>\n";
+    print "* OPTIONAL: a datastore product name            -p <name>\n";
 }
 if (!defined $singleBatch) {
-    print "* OPTIONAL: a single batch                  -b <batchNum>        (default = none)\n";
+    print "* OPTIONAL: a single batch                      -b <batchNum>        (default = none)\n";
 }
 if (!defined $fromTime) {
     $fromTime = "2010-01-01";
-    print "* OPTIONAL: from time                       -f <dateTime>        (default = $fromTime)\n";
+    print "* OPTIONAL: from time                           -f <dateTime>        (default = $fromTime)\n";
 }
 if (!defined $toTime) {
     $toTime = "2099-12-31";
-    print "* OPTIONAL: to time                         -t <dateTime>        (default = $toTime)\n";
+    print "* OPTIONAL: to time                             -t <dateTime>        (default = $toTime)\n";
 }
 if (!defined $filePath) {
     print "* OPTIONAL: location for files to be deleted    -l <path>\n";
 }
+if (!defined $removeUnprocessed) {
+    $removeUnprocessed = 0;
+    print "* OPTIONAL: remove unprocessed files            -r                   (default = $removeUnprocessed)\n";
+}
 if (!defined $verbose) {
     $verbose = 0;
-    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+    print "* OPTIONAL: run in verbose mode                 -v                   (default = $verbose)\n";
 }
 if (!defined $save_temps) {
     $save_temps = 0;
-    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+    print "* OPTIONAL: keep temp files                     -t                   (default = $save_temps)\n";
 }
 
@@ -60,5 +66,4 @@
 my $datastore = new ippToPsps::Datastore($product, 0, 0);
 my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
-my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $filePath, $verbose, $save_temps);
 my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus";
 my $ua = LWP::UserAgent->new;
@@ -85,7 +90,7 @@
 
     print "\n";
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
-    printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Merge worthy? | Merged? | Deleted? |\n");
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
+    printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Load failed? |Merge worthy? | Merged? | Deleted? |\n");
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
 
     # loop round batches
@@ -95,23 +100,58 @@
     my $numRemovedFromDatastore = 0;
     my $numDeleted = 0;
+    my $newMerged = 0;
     foreach $batch ( @{$batches} ) {
-        my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
-
-        if (!$processed) {next;}
+        my ($timestamp, 
+                $type, 
+                $expId, 
+                $batchId, 
+                $surveyType,
+                $deleted,
+                $dvoDb, 
+                $processed, 
+                $onDatastore, 
+                $loadedToOdm, 
+                $loadFailed, 
+                $mergeWorthy, 
+                $merged) = @{$batch};
+
+        if (!$onDatastore) {next;}
+
+        my $batch = ippToPsps::Batch->existing(
+                undef,
+                undef,
+                $batchId, 
+                $ippToPspsDb, 
+                $filePath, 
+                $verbose, 
+                $save_temps);
+
+        if (!$processed) {
+
+            if ($removeUnprocessed && defined $filePath) {
+
+                if ($batch->deleteDir()) {$numDeleted++;}
+            }
+            next;
+        }
 
         $numBatchesToCheck++;
 
-        my $batchName = $batchManager->getBatchName($batchId);
-
         # if not merged then update by polling ODM for status
-        if (!$merged) {
-
-            if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;}
+        if (!$merged && !$loadFailed) {
+
+            if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) {
+                
+                $numChecked++;
+                if ($merged) {$newMerged++;}
+            }
+            else {next;}
         }
 
         # delete from datastore
-        if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) {
-
-            $deleted = $datastore->remove($batchName);
+        if (defined $product && !$deleted && $loadedToOdm && ($mergeWorthy || $loadFailed)) {
+
+
+            $deleted = $datastore->remove($batch->getName());
             if ($deleted) {
                 $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
@@ -122,14 +162,15 @@
         if (defined $filePath && $merged && $deleted) {
 
-            if($batchManager->deleteBatch($batchId)) {$numDeleted++;}
+            if($batch->deleteFile($batchId)) {$numDeleted++;}
         }
 
         # update database
-        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted);
-        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n", 
+        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $loadFailed, $mergeWorthy, $merged, $deleted);
+        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s    |   %6s     | %6s  |  %5s   |\n", 
                 $timestamp, 
-                $batchName, 
+                $batch->getName(), 
                 $expId, 
                 $loadedToOdm ? "yes" : "no", 
+                $loadFailed ? "yes" : "no", 
                 $mergeWorthy ? "yes" : "no", 
                 $merged ? "yes" : "no",
@@ -137,7 +178,7 @@
     }
 
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
-
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
     printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck);
+    printf( "* %d newly merged batches\n", $newMerged);
     printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es");
     printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es");
@@ -151,5 +192,5 @@
 ########################################################################################
 sub checkODM {
-    my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_;
+    my ($batchName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
 
     my $statusFilter = "*";
@@ -171,17 +212,21 @@
 
     my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_odmXml.XXXX", UNLINK => !$save_temps);
+    #print $response->content. "\n";
     print $tempFile $response->content;
     close($tempFile);
 
     ${$loadedToOdm} = 0;
+    ${$loadFailed} = 0;
     ${$mergeWorthy} = 0;
     ${$merged} = 0;
 
-    parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged);
+    parseXml($tempName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged);
+
+    unlink($tempFile);
 
     return 1;
 }
 
-######################################################################################y
+########################################################################################
 # 
 # Parses ODM XML
@@ -189,5 +234,5 @@
 ########################################################################################
 sub parseXml {
-    my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_;
+    my ($xmlFile, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
 
     my $parser = XML::LibXML->new;
@@ -195,5 +240,13 @@
     my $xc = XML::LibXML::XPathContext->new($doc);
     $xc->registerNs('ArrayOfOdmBatchState', 'PanSTARRS.Services.OdmWebService');
-    my $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
+    my $result = $xc->findvalue('//ArrayOfOdmBatchState:BatchState');
+
+    if ($result =~ m/failed/) { 
+        ${$loadFailed} = 1;
+        ${$loadedToOdm} = 1;
+    return;
+    }
+
+    $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
     ${$loadedToOdm} = 0;
     ${$mergeWorthy} = 0;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl	(revision 29515)
@@ -7,13 +7,13 @@
 use ippToPsps::Gpc1Db;
 use ippToPsps::IppToPspsDb;
-use PS::IPP::Config 1.01 qw( :standard );
+use ippToPsps::DetectionBatch;
 
 my $camera = undef;
-my $singleBatch = undef;
+my $batch = undef;
 my $verbose = undef;
 my $save_temps = undef;
 
 GetOptions(
-        'batch|b=s' => \$singleBatch,
+        'batch|b=s' => \$batch,
         'verbose|v' => \$verbose,
         'save_temps|t' => \$save_temps,
@@ -26,11 +26,11 @@
 if (@ARGV) {
     $quit=1;
-    print "* UNKNKOWN: option                          @ARGV\n";
+    print "* UNKNOWN: option                          @ARGV\n";
 }
-if (!defined $singleBatch) {
+if (!defined $batch) {
     $quit=1;
     print "* REQUIRED: need to provide a batch   -b <batch>\n";
 }
-if (!defined $verbose) {
+if (!defined $camera) {
     $camera = "GPC1";
     print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
@@ -51,32 +51,19 @@
 my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
 my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
-my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR));
 
+my $detectionBatch = ippToPsps::DetectionBatch->existing(
+        $camera,
+        $gpc1Db,
+        $batch, 
+        $ippToPspsDb, 
+        "./",
+        $verbose, 
+        $save_temps);
 
-my $batches;
-my $numOfBatches;
-
-if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);}
-if ($numOfBatches < 1) {return 0;}
-
-
-# loop round batches
-my $batch;
-my $numChecked = 0;
-foreach $batch ( @{$batches} ) {
-    my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
-
-    print "* found $timestamp, $expId, $batchId, $surveyType, $deleted $dvoDb\n";
-
-    my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId);
-    if (!$nebPath) { print "* Could not determind neb path for SMF\n"; exit; }
-
-    # get real filename from neb 'key'
-    my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0;
-    my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
-
-    print "$realFile\n";
-
+my $smf = $detectionBatch->getSmfFile();
+if (defined $smf) {
+    print "$smf\n";
 }
-
-
+else {
+    print "* Could not find an smf file for batch $batch\n";
+}
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Batch.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Batch.pm	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Batch.pm	(revision 29515)
@@ -0,0 +1,523 @@
+#!/usr/bin/perl -w
+
+package ippToPsps::Batch;
+
+use warnings;
+use strict;
+use XML::LibXML;
+use PS::IPP::Config 1.01 qw( :standard );
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Basename;
+use File::Temp qw(tempfile);
+
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub existing {
+
+    my $class = shift;
+    my $self = {
+        _camera => shift,
+        _gpc1Db => shift,
+        _batchId => shift,
+        _ippToPspsDb => shift,
+        _baseDir => shift,
+        _verbose => shift,
+        _save_temps => shift,
+    };
+
+    my $batches;
+    $self->{_ippToPspsDb}->getBatch($self->{_batchId}, \$batches);
+    my $batch;
+    foreach $batch ( @{$batches} ) {
+        my ($timestamp, 
+                $type,
+                $expId, 
+                $batchId, 
+                $surveyType, 
+                $deleted,
+                $dvoDb, 
+                $processed, 
+                $onDatastore,
+                $loadedToOdm,
+                $loadFailed,
+                $mergeWorthy, 
+                $merged) = @{$batch};
+
+        $self->{_timestamp} = $timestamp; 
+        $self->{_type} = $type; 
+        $self->{_expId} = $expId; 
+        $self->{_batchId} = $batchId; 
+        $self->{_surveyType} = $surveyType; 
+        $self->{_deleted} = $deleted; 
+        $self->{_dvoDb} = $dvoDb; 
+        $self->{_processed} = $processed; 
+        $self->{_onDatastore} = $onDatastore; 
+        $self->{_loadedToOdm} = $loadedToOdm; 
+        $self->{_mergeWorthy} = $mergeWorthy; 
+        $self->{_merged} = $merged;
+    }
+
+    bless $self, $class;
+
+    if (!defined $self->{_baseDir}) {$self->{_baseDir} = "./";}
+    $self->{_parentDir} = $self->{_dvoDb};
+
+    $self->init();
+    return $self;
+}
+
+###########################################################################
+#
+# Constructor
+#
+# This is the superclass of all different types of Batches, eg
+# - detections (DetectionBatch)
+# - init (InitBatch)
+# - etc
+#
+###########################################################################
+sub create {
+
+    my $class = shift;
+    my $self = {
+        _type => shift,        # batch type, IN,P2,ST...
+            _camera => shift,
+        _gpc1Db => shift,
+        _ippToPspsDb => shift,
+        _baseDir => shift,        # output path
+            _datastore => shift,
+        _expId => shift,
+        _expName => shift,
+        _dvoDb => shift,
+        _dvoPath => shift,
+        _distGroup => shift,
+        _dontTarball => shift,
+        _verbose => shift,
+        _save_temps => shift,
+    };
+
+    bless $self, $class;
+
+    # break up the DVO Db path, if provided
+    if (defined $self->{_dvoPath} && defined $self->{_dvoDb}) {
+
+        $self->{_parentDir} = $self->{_dvoDb};
+    }
+    else {
+
+        $self->{_parentDir} = ".";
+        $self->{_dvoDb} = "NULL";
+    }
+
+    # create a new batch entry in the database
+    $self->{_batchId} = $self->{_ippToPspsDb}->createNewBatch(
+            $self->{_expId}, 
+            $self->{_distGroup}, 
+            $self->{_type}, 
+            $self->{_dvoDb}, 
+            (defined $self->{_datastore}) ? $self->{_datastore}->getProduct() : "NONE"
+            );
+
+    # use IPP dist_group to get PSPS survey type
+    $self->{_surveyType} = $self->getSurveyTypeFromDistGroup();
+
+    # can we run ippToPsps?
+    $self->{_ippToPsps} = can_run('ippToPsps') or
+        (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
+
+    # set up batch output paths
+    $self->init();
+
+    # make directories
+    unless(-d $self->{_typeDir}) {mkdir($self->{_typeDir}, 0777);}
+    unless(-d $self->{_subDir}) {mkdir($self->{_subDir}, 0777);}
+    mkdir($self->{_batchDir}, 0777);
+
+    return $self;
+}
+
+########################################################################################
+# 
+# Sets names and paths
+# 
+########################################################################################
+sub init {
+    my ($self) = @_;
+
+    # get ipp config stuff
+    if (defined $self->{_camera}) {
+
+        $self->{_ipprc} = PS::IPP::Config->new($self->{_camera}) or
+            (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR));
+    }
+
+    $self->{_name} =  sprintf("B%08d", $self->{_batchId});
+    $self->{_filename} = "$self->{_name}.tar.gz"; 
+    $self->{_typeDir} = "$self->{_baseDir}/$self->{_type}";
+    $self->{_subDir} = "$self->{_typeDir}/$self->{_parentDir}";
+    $self->{_batchDir} = "$self->{_subDir}/$self->{_name}";
+    $self->{_tarball} = "$self->{_subDir}/$self->{_filename}";
+
+    #print "name     $self->{_name}\n";
+    #print "filename $self->{_filename}\n";
+    #print "subDir   $self->{_subDir}\n";
+    #print "batchDir $self->{_batchDir}\n";
+    #print "tarball  $self->{_tarball}\n";
+
+}
+
+#######################################################################################
+# 
+# Generates a PSPS-suitable survey 'type' from the IPP distribution group
+#
+#######################################################################################
+sub getSurveyTypeFromDistGroup {
+    my ($self) = @_;
+
+    if ($self->{_distGroup} =~ m/^MD([0-1][0-9])$/i) {return "MD$1";}
+    if ($self->{_distGroup} =~ m/^M31$/i) {return "M31";}
+    if ($self->{_distGroup} =~ m/^sts$/i) {return "STS1";}
+    if ($self->{_distGroup} =~ m/^SweetSpot$/i) {return "SSS";}
+    if ($self->{_distGroup} =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";}
+
+    print "* Unknown distribution group: '$self->{_distGroup}'\n";
+    return $self->{_distGroup};
+}
+
+#######################################################################################
+#
+# runs the ippToPsps program
+#
+#######################################################################################
+sub runProgram {
+    my ($self, $input, $resultsFilePath) = @_;
+
+    # build command
+    my $command = "$self->{_ippToPsps}";
+    $command .= " -input $input";
+    $command .= " -output $self->{_batchDir}";
+    if (defined $self->{_dvoPath} ) {$command .= " -D CATDIR $self->{_dvoPath}";}
+    $command .= " -config ../config"; # TODO
+        $command .= " -id $self->{_expId}";
+    $command .= " -expname $self->{_expName}";
+    $command .= " -survey $self->{_surveyType}";
+    $command .= " -batch $self->{_type}";
+    $command .= " -results $resultsFilePath";
+
+    print "*\n*******************************************************************************\n\n";
+
+    # run command
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
+        = run(command => $command, verbose => $self->{_verbose});
+
+    print "\n*******************************************************************************\n\n";
+
+    return $success;
+}
+
+
+########################################################################################
+# 
+# Returns batch ID
+# 
+########################################################################################
+sub getId {
+    my ($self) = @_;
+    return $self->{_batchId};
+}
+
+########################################################################################
+# 
+# Returns batch name
+# 
+########################################################################################
+sub getName {
+    my ($self) = @_;
+    return $self->{_name};
+}
+
+########################################################################################
+# 
+# Returns name of tarred and zipped batch file
+# 
+########################################################################################
+sub getFileName {
+    my ($self) = @_;
+    return $self->{_filename};
+}
+
+########################################################################################
+# 
+# Returns full path of type dir
+# 
+########################################################################################
+sub getTypeDir {
+    my ($self) = @_;
+    return $self->{_typeDir};
+}
+
+########################################################################################
+# 
+# Returns full path of sub dir
+# 
+########################################################################################
+sub getSubDir {
+    my ($self) = @_;
+    return $self->{_subDir};
+}
+
+########################################################################################
+# 
+# Returns full path of batch dir
+# 
+########################################################################################
+sub getBatchDir {
+    my ($self) = @_;
+    return $self->{_batchDir};
+}
+
+########################################################################################
+# 
+# Returns file name of tarball 
+# 
+########################################################################################
+sub getFilename {
+    my ($self) = @_;
+    return $self->{_filename};
+}
+
+########################################################################################
+# 
+# Returns full path of tarball 
+# 
+########################################################################################
+sub getTarball {
+    my ($self) = @_;
+    return $self->{_tarball};
+}
+
+########################################################################################
+# 
+# Deletes a batch tar.gz file
+# 
+########################################################################################
+sub deleteFile {
+    my ($self) = @_;
+
+    if (! -e $self->{_tarball}) {return 0;}
+
+    my $command = "rm " . $self->{_tarball};
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) {
+        print "* Unable to run: $command\n";
+        return 0;
+    }
+
+    return 1;
+}
+
+########################################################################################
+# 
+# Deletes batch dir 
+# 
+########################################################################################
+sub deleteDir {
+    my ($self) = @_;
+
+    if (! -e $self->{_batchDir}) {return 0;}
+    my $command = "rm -r " . $self->{_batchDir};
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) {
+        print "* Unable to run: $command\n";
+        return 0;
+    }
+
+    return 1;
+}
+
+#######################################################################################
+#
+# Writes the manifest file
+#
+#######################################################################################
+sub writeManifest {
+    my ($self, $filename, $minObjId, $maxObjId) = @_;
+
+    use XML::Writer;
+    use Digest::MD5::File qw( file_md5_hex );
+
+    # sort out paths
+    my $outputPath = "$self->{_batchDir}/BatchManifest.xml";
+    my $output = new IO::File(">$outputPath");
+
+    # create a timestamp
+    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
+    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
+
+    # create XML file
+    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
+    $writer->xmlDecl('UTF-8');
+
+    # get md5sum of file
+    my $md5sum  = file_md5_hex("$self->{_batchDir}/$filename");
+
+    # get size of file in bytes
+    my $filesize = -s "$self->{_batchDir}/$filename";
+
+    # write manifest element TODO needs to be in subclass
+    if($self->{_type} eq 'P2') {
+
+        my $pspsSurvey = undef;
+        if ($self->{_surveyType} =~ m/^MD[0-1][0-9]$/i) {$pspsSurvey = "MDF";}
+        else {$pspsSurvey = $self->{_surveyType};}
+
+        $writer->startTag('manifest',
+                "name" => "$self->{_name}",
+                "type" => "$self->{_type}",
+                "survey" => $pspsSurvey,
+                "timestamp" => "$timeStamp",
+                "minObjId" => $minObjId,
+                "maxObjId" => $maxObjId);
+    }
+    else {
+
+        print "'$self->{_name}' '$self->{_type}' '$timeStamp'\n";
+
+        $writer->startTag('manifest',
+                "name" => "$self->{_name}",
+                "type" => "$self->{_type}",
+                "timestamp" => "$timeStamp");
+    }
+    # write the tag for this batch
+    $writer->startTag("file",
+            "name" => $filename,
+            "bytes" => $filesize,
+            "md5" => $md5sum);
+
+    $writer->endTag();
+    $writer->endTag();
+    $writer->end();
+
+    return 1;
+}
+
+#######################################################################################
+#
+# tar 'n zip a batch TODO handle errors
+#
+#######################################################################################
+sub createTarball {
+    my ($self) = @_;
+
+    my $tar = $self->{_name}.".tar";
+    my $tarball = "$tar.gz";
+    my $batchTar = "$self->{_batchDir}/$tar";
+
+    # tar
+    my $command = "tar -cvf $batchTar -C $self->{_subDir} $self->{_name}";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) { print "* Unable to tar up contents of dir: ". $self->{_batchDir}. "\n" and return 0; }
+
+    # zip
+    $command = "gzip -c $batchTar  > $self->{_tarball}";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0; }
+
+    # remove tar file
+    $command = "rm $batchTar";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) { print "* Unable to remove: $batchTar\n" };
+
+    # remove batch dir
+    $command = "rm -r $self->{_batchDir}";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $self->{_verbose});
+    if (!$success) { print "* Unable to remove: $self->{_name}\n"};
+
+    return 1;
+}
+
+#######################################################################################
+#
+# Abstract method to be inherited by subclasses
+#
+#######################################################################################
+sub process {
+    my ($self) = @_;
+
+    print "* Preparing batch '$self->{_name}' for '$self->{_type}'\n";
+
+    # create temporary input file for file list
+    my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$self->{_save_temps});
+
+    # get list of input files
+    my ($inputFile, $inputPath) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$self->{_save_temps});
+
+    my $success = 0;
+    if ($self->createInputFileList($inputFile)) {
+
+        # run IppToPsps program TODO should not need
+        $success = $self->runProgram($inputPath, $resultsFilePath);
+
+        if ($success) {
+
+            # TODO type-specific processing
+            $self->parseResults($resultsFilePath);
+
+            # tar n' zip
+            if (!$self->{_dontTarball} && $self->createTarball()) {
+
+                # and publish
+                if (defined $self->{_datastore} &&
+                        $self->{_datastore}->register($self->{_name}, $self->{_subDir}, $self->{_filename}, "IPP_PSPS", "tgz")) {
+                    $self->{_ippToPspsDb}->setPublished($self->{_batchId}, $self->{_expId}, 1);
+                }
+            }
+        }
+    }
+
+    close $resultsFile;
+    close $inputFile;
+
+    if (!$self->{_save_temps}) {unlink $resultsFile};
+    if (!$self->{_save_temps}) {unlink $inputFile};
+
+    return $success;
+}
+
+#######################################################################################
+# 
+# Abstract method to be inherited by subclasses
+# Creates an input file list for ippToPsps 
+#
+#######################################################################################
+sub createInputFileList {
+    my ($self) = @_;
+    my $thisFunction = (caller(0))[3];
+    print "* ERROR: '$thisFunction' not implemented\n";
+}
+
+#######################################################################################
+#
+# Abstract method to be inherited by subclasses
+# Deals with the results
+#
+#######################################################################################
+sub parseResults {
+    my ($self) = @_;
+
+    my $thisFunction = (caller(0))[3];
+    print "* ERROR: '$thisFunction' not implemented\n";
+}
+
+1;
Index: anches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/BatchManager.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/BatchManager.pm	(revision 29514)
+++ 	(revision )
@@ -1,95 +1,0 @@
-#!/usr/bin/perl -w
-
-package ippToPsps::BatchManager;
-
-use warnings;
-use strict;
-
-use IPC::Cmd 0.36 qw( can_run run );
-
-###########################################################################
-#
-# Constructor
-#
-###########################################################################
-sub new {
-
-    my $class = shift;
-    my $self = {
-        _ippToPspsDb => shift,
-        _path => shift,
-        _verbose => shift,
-        _save_temps => shift,
-    };
-
-    bless $self, $class;
-    return $self;
-}
-
-########################################################################################
-# 
-# Returns batch name from batch_id
-# 
-########################################################################################
-sub getBatchName {
-    my ($self, $batchId) = @_;
-
-    return sprintf("B%08d", $batchId);
-}
-
-########################################################################################
-# 
-# Returns name of tarred and zipped batch file
-# 
-########################################################################################
-sub getBatchFileName {
-    my ($self, $batchId) = @_;
-
-    my $batchName = $self->getBatchName($batchId);
-    return "$batchName.tar.gz";
-}
-
-########################################################################################
-# 
-# Returns full path of batch 
-# 
-########################################################################################
-sub getFullBatchPath {
-    my ($self, $batchId) = @_;
-
-    my $batches;
-    my $numOfBatches = $self->{_ippToPspsDb}->getBatch($batchId, \$batches);
-    if ($numOfBatches < 1) {return "unknown";}
-
-    my $batch;
-    my ($timestamp, $expId, $batchIdi, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged);
-    foreach $batch ( @{$batches} ) {
-        ($timestamp, $expId, $batchIdi, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
-
-
-    }
-
-    my $batchFile = $self->getBatchFileName($batchId);
-    return "$self->{_path}/$dvoDb/$batchFile";
-}
-
-########################################################################################
-# 
-# Deletes a batch 
-# 
-########################################################################################
-sub deleteBatch {
-    my ($self, $batchId) = @_;
-
-    my $command = "rm " . $self->getFullBatchPath($batchId);
-    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
-        run(command => $command, verbose => $self->{_verbose});
-    if (!$success) {
-        print "* Unable to run: $command\n";
-        return 0;
-    }
-
-    return 1;
-}
-
-1;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Datastore.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Datastore.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Datastore.pm	(revision 29515)
@@ -27,4 +27,14 @@
     bless $self, $class;
     return $self;
+}
+
+########################################################################################
+# 
+# Returns datastore product
+# 
+########################################################################################
+sub getProduct {
+    my ($self) = @_;
+    return $self->{_product};
 }
 
@@ -58,5 +68,6 @@
     print "* Successfully published $file to datastore\n";
 
-    close($tempFile);
+    close $tempFile;
+    if (!$self->{_save_temps}) {unlink $tempFile};
 
     return 1;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/DetectionBatch.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/DetectionBatch.pm	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/DetectionBatch.pm	(revision 29515)
@@ -0,0 +1,134 @@
+#!/usr/bin/perl -w
+
+package ippToPsps::DetectionBatch;
+
+use warnings;
+use strict;
+use File::Basename;
+use File::Temp qw(tempfile);
+use XML::LibXML;
+
+use ippToPsps::Batch;
+our @ISA = qw(ippToPsps::Batch);    # inherits from Batch class
+
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub existing {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::existing(
+            $_[1],  # camera
+            $_[2],  # gpc1Db
+            $_[3],  # batchId
+            $_[4],  # ippToPspsDb
+            $_[5],  # path
+            $_[6],  # verbose
+            $_[7]   # save_temps
+            );
+
+    bless $self, $class;
+    return $self;
+}
+
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::create(
+            "P2",   # type 
+            $_[1],  # camera
+            $_[2],  # gpc1Db
+            $_[3],  # ippToPspsDb
+            $_[4],  # path
+            $_[5],  # datastore
+            $_[6],  # expId
+            $_[7],  # expName
+            $_[8],  # dvoDb
+            $_[9],  # dvoPath
+            $_[10], # distGroup
+            $_[11], # dontTarBall
+            $_[12], # verbose
+            $_[13]  # save_temps
+            );
+
+    bless $self, $class;
+    $self->process();
+    return $self;
+}
+
+#######################################################################################
+# 
+# Locates the relevant smf file for this exposure
+#
+#######################################################################################
+sub getSmfFile {
+    my ($self) = @_;
+
+    # get neb path of smf file
+    my $nebPath = $self->{_gpc1Db}->getCameraStageSmfForThisDvoDb($self->{_dvoDb}, $self->{_expId});
+    if (!$nebPath) { return 0; }
+
+    # get real filename from neb 'key'
+    my $fpaObjects = $self->{_ipprc}->filename("PSASTRO.OUTPUT",$nebPath) or return undef;
+
+    return $self->{_ipprc}->file_resolve($fpaObjects);
+}
+
+#######################################################################################
+# 
+# Locates the relevant smf file for this exposure and writes the path to the file passed-in
+#
+#######################################################################################
+sub createInputFileList {
+    my ($self, $inputFile) = @_;
+
+    my $smfFile = $self->getSmfFile();
+    if (!defined $smfFile) {return 0;}
+
+    # now write the path to this file out to temp file
+    print $inputFile $smfFile . "\n";
+
+    return 1;
+}
+
+#######################################################################################
+#
+# - Reads results from processing (min/max obj ID etc)
+# - writes to Db
+# - writes manifest file
+#
+#######################################################################################
+sub parseResults {
+    my ($self, $resultsFilePath) = @_;
+
+    my($filename, $minObjId, $maxObjId, $totalDetections);
+
+    # read results of FITS creation from XML file
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file($resultsFilePath);
+    $filename = $doc->findvalue('//filename');
+    $minObjId = $doc->findvalue('//minObjID');
+    $maxObjId = $doc->findvalue('//maxObjID');
+    $totalDetections = $doc->findvalue('//totalDetections');
+
+    # write results to database
+    $self->{_ippToPspsDb}->setAsProcessed($self->{_batchId}, $self->{_expId}, $totalDetections, $minObjId, $maxObjId);
+
+    # write batch manifest
+    if (!$self->writeManifest($filename, $minObjId, $maxObjId)) {return 0;}
+
+    return 1;
+}
+1;
+
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Gpc1Db.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Gpc1Db.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/Gpc1Db.pm	(revision 29515)
@@ -8,4 +8,31 @@
 use ippToPsps::MySQLDb;
 our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
+
+###########################################################################
+#
+# Returns path to stack-stage cmf files for this sky_id
+#
+###########################################################################
+sub getStackStageCmfs {
+    my ($self, $skyId, $nebPath, $numInputs) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT path_base, num_inputs
+        FROM staticskyResult
+        WHERE sky_id = $skyId
+SQL
+
+    $query->execute;
+    my $rows = $query->fetchall_arrayref();
+
+    if (@{$rows} < 1 || @{$rows} > 1) {return 0;}
+
+    my $row;
+    foreach $row ( @{$rows} ) {
+        (${$nebPath}, ${$numInputs}) = @{$row};
+    }
+
+    return 1;
+}
 
 ###########################################################################
@@ -90,5 +117,7 @@
 ###########################################################################
 sub getExposureListFromDvoDb {
-    my ($self, $dvoDb, $exposures) = @_;
+    my ($self, $dvoDb, $exposures, $lastExposure) = @_;
+
+    if (!defined $lastExposure) {$lastExposure = 0;}
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -100,14 +129,8 @@
         WHERE addRun.dvodb LIKE '$dvoDb' 
         AND addRun.state = 'full'
+        AND rawExp.exp_id > $lastExposure
         ORDER BY exp_id ASC;
 SQL
 
-    #AND rawExp.exp_id > 133887
-    #AND filter = 'r.00000'
-    #AND rawExp.dateobs <= '2010-03-12'
-    #AND rawExp.exp_id > $lastExpId
-    #AND camRun.label LIKE '%$label%'
-    #AND rawExp.exp_id > 136561
-    #AND camRun.dist_group LIKE 'sas'
     #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
 
@@ -115,12 +138,11 @@
     ${$exposures} = $query->fetchall_arrayref();
     my $numOfExposures = scalar @{${$exposures}};
-    if ($numOfExposures > 0) {
-    
-        print "* Found $numOfExposures exposures in DVO Db '$dvoDb'\n";
-        return 1;
-    }
 
-    print "* No exposures found in DVO Db '$dvoDb'\n";
-    return 0;
+    # message
+    if ($numOfExposures) {print "* Found $numOfExposures exposures";}
+    else { print "* No exposures found"; }
+    print " in DVO Db '$dvoDb' (counting from exposure $lastExposure)\n";
+
+    return ($numOfExposures) ? 1 : 0;
 }
 
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/InitBatch.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/InitBatch.pm	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/InitBatch.pm	(revision 29515)
@@ -0,0 +1,81 @@
+#!/usr/bin/perl -w
+
+package ippToPsps::InitBatch;
+
+use warnings;
+use strict;
+use File::Basename;
+use File::Temp qw(tempfile);
+use XML::LibXML;
+
+use ippToPsps::Batch;
+our @ISA = qw(ippToPsps::Batch);    # inherits from Batch class
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::create(
+            "IN",      # type 
+            $_[1],     # camera
+            undef,     # gpc1Db
+            $_[2],     # ippToPspsDb
+            $_[3],     # path
+            $_[4],     # datastore
+            0,         # expId
+            "NULL",    # expName
+            undef,     # dvoDb
+            undef,     # dvoPath
+            "ThreePi", # distGroup
+            $_[5],     # dontTarball
+            $_[6],     # verbose
+            $_[7]);    # save_temps
+
+    bless $self, $class;
+    $self->process();
+    return $self;
+}
+
+#######################################################################################
+# 
+# No input file necessary for IN batches 
+#
+#######################################################################################
+sub createInputFileList {
+    my ($self) = @_;
+
+    return "NULL";
+}
+
+#######################################################################################
+#
+# - Reads filename from results XML file
+# - Updates Db
+# - write manifest file
+#
+#######################################################################################
+sub parseResults {
+    my ($self, $resultsFilePath) = @_;
+
+    my($filename, $minObjId, $maxObjId, $totalDetections);
+
+    # read results of FITS creation from XML file
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file($resultsFilePath);
+    $filename = $doc->findvalue('//filename');
+
+    # write results to database
+    $self->{_ippToPspsDb}->setAsProcessed($self->{_batchId}, $self->{_expId}, -1, -1, -1);
+
+    # write batch manifest
+    if (!$self->writeManifest($filename, -1, -1)) {return 0;}
+
+    return 1;
+}
+1;
+
Index: anches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPsps.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPsps.pm	(revision 29514)
+++ 	(revision )
@@ -1,43 +1,0 @@
-#!/usr/bin/perl -w
-
-use warnings;
-use strict;
-use DBI;
-
-package ippToPsps::IppToPsps;
-
-###########################################################################
-#
-# Constructor
-#
-###########################################################################
-sub new {
-
-    my $class = shift;
-    my $self = {
-        _dbName => shift,
-        _dbHost => shift,
-        _dbUser => shift,
-        _dbPass => shift,
-        _verbose => shift,
-    };
-
-    my $dbname = $self->{_dbName};
-    my $dbhost = $self->{_dbHost};
-    my $dbuser = $self->{_dbUser};
-    my $dbpass = $self->{_dbPass};
-
-    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
-
-    $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
-            "mysql_socket=" . DB_SOCKET(),
-            ${dbuser},${dbpass},
-            { RaiseError => 1, AutoCommit => 1}
-            );
-
-    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
-
-    bless $self, $class;
-    return $self;
-}
-
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm	(revision 29515)
@@ -9,5 +9,4 @@
 our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
 
-
 ###########################################################################
 #
@@ -19,8 +18,20 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged 
+    SELECT created, 
+           batch_type, 
+           exp_id, 
+           batch_id, 
+           survey_id, 
+           deleted, 
+           dvo_db, 
+           processed, 
+           on_datastore, 
+           loaded_to_ODM, 
+           load_failed, 
+           merge_worthy, 
+           merged 
         FROM batches 
         WHERE created >= '$fromTime'
-        AND created <= '$toTime';
+        AND created <= '$toTime'
 SQL
 
@@ -30,5 +41,4 @@
     my $count = scalar @{${$batches}};
 
-   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    return $count;
 }
@@ -43,5 +53,17 @@
 
     my $query = $self->{_db}->prepare(<<SQL);
-    SELECT created, exp_id, batch_id, survey_id, deleted, dvo_db, processed, on_datastore, loaded_to_ODM, merge_worthy, merged
+    SELECT created, 
+           batch_type, 
+           exp_id, 
+           batch_id, 
+           survey_id, 
+           deleted, 
+           dvo_db, 
+           processed, 
+           on_datastore, 
+           loaded_to_ODM, 
+           load_failed, 
+           merge_worthy, 
+           merged
         FROM batches 
         WHERE batch_id = $batch_id 
@@ -52,5 +74,4 @@
     my $count = scalar @{${$batches}};
 
-   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
    return $count;
 }
@@ -103,6 +124,6 @@
 #
 ########################################################################################
-sub updateBatch {
-    my ($self, $batchId, $expId, $processed, $published, $totalDetections, $minObjId, $maxObjId) = @_;
+sub setAsProcessed {
+    my ($self, $batchId, $expId, $totalDetections, $minObjId, $maxObjId) = @_;
 
     if (!$minObjId) {$minObjId = -1;}
@@ -114,6 +135,5 @@
     UPDATE batches 
         SET 
-          processed = $processed, 
-          on_datastore = $published, 
+          processed = 1, 
           total_detections = $totalDetections,
           min_obj_id = $minObjId,
@@ -128,8 +148,26 @@
 #######################################################################################
 #
-# Checks whether we have successfully processed this exposure 
-#
-########################################################################################
-sub isExposureAlreadyProcessed {
+# Sets published state of a batch 
+#
+########################################################################################
+sub setPublished {
+    my ($self, $batchId, $expId, $published) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    UPDATE batches 
+        SET on_datastore = $published 
+        WHERE batch_id = $batchId 
+        AND exp_id = $expId;
+SQL
+
+    $query->execute; # TODO check response of this
+}
+
+#######################################################################################
+#
+# Checks whether we have successfully processed this exposure and loaded it to the datastore 
+#
+########################################################################################
+sub isExposureAlreadyPublished {
     my ($self, $expId) = @_;
 
@@ -138,6 +176,6 @@
         FROM batches 
         WHERE exp_id = $expId
-        AND created > '2010-08-12'
-        AND processed = 1;
+        AND processed
+        AND on_datastore;
 SQL
 
@@ -179,5 +217,5 @@
     $query->execute;
 
-    print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n";
+    print "* New batch '$batchId' for exposure ID = '$expId' and survey = '$surveyType'\n";
 
     return $batchId;
@@ -208,9 +246,12 @@
 #######################################################################################
 sub updateODMStatus {
-    my ($self,$batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_;
+    my ($self,$batchId, $expId, $loadedToOdm, $loadFailed, $mergeWorthy, $mergeCompleted) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     UPDATE batches 
-        SET loaded_to_ODM = $loadedToOdm, merge_worthy = $mergeWorthy, merged = $mergeCompleted 
+        SET loaded_to_ODM = $loadedToOdm, 
+            load_failed = $loadFailed, 
+            merge_worthy = $mergeWorthy, 
+            merged = $mergeCompleted 
         WHERE batch_id = $batchId 
         AND exp_id = $expId;
@@ -231,7 +272,6 @@
 
     my $currentRevision = -1;
-    my $latestRevision = 6;
-
-    while ($currentRevision != $latestRevision) {
+
+    while (1) {
 
         $currentRevision = $self->getRevision();
@@ -244,4 +284,6 @@
         elsif ($currentRevision == 4) {$self->createRevision_5();}
         elsif ($currentRevision == 5) {$self->createRevision_6();}
+        elsif ($currentRevision == 6) {$self->createRevision_7();}
+        else {last;}
     }
 }
@@ -385,4 +427,23 @@
 #######################################################################################
 # 
+# Create revision 7 of the database 
+#
+#######################################################################################
+sub createRevision_7 {
+    my ($self) = @_;
+
+    print "* Creating revision 7 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE batches 
+        ADD COLUMN load_failed TINYINT DEFAULT 0
+SQL
+        $query->execute;
+
+    $self->setRevision(7);
+}
+
+#######################################################################################
+# 
 # Sets current revision of ippToPsps database
 #
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/MySQLDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/MySQLDb.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/MySQLDb.pm	(revision 29515)
@@ -65,15 +65,14 @@
 }                                                                               
 
-
 ###########################################################################
 #
-# Returns 'now' as a timestamp
+# Adds the provided interval to the provided time
 #
 ###########################################################################
-sub getIntervalInPast {
-    my ($self, $interval) = @_;
+sub addInterval {
+    my ($self, $time, $interval) = @_;
 
       my $query = $self->{_db}->prepare(<<SQL);
-          SELECT now() - INTERVAL $interval; 
+          SELECT '$time' + INTERVAL $interval; 
 SQL
     $query->execute;
@@ -82,4 +81,35 @@
 }
 
+###########################################################################
+#
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns whether time1 is before time2
+#
+###########################################################################
+sub isBefore {
+    my ($self, $time1, $time2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time1' < '$time2'; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
 
 ###########################################################################
@@ -97,4 +127,25 @@
 
 return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Optimizes a table 
+#
+#######################################################################################
+sub optimizeTable {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    OPTIMIZE TABLE $table; 
+SQL
+
+    my $success = $query->execute;
+
+    print "* ";
+    if (!$success) {print "UN";}
+    print "successfully optimized '$table' table\n";
+
+    return $success;
 }
 
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/StackBatch.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/StackBatch.pm	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/StackBatch.pm	(revision 29515)
@@ -0,0 +1,130 @@
+#!/usr/bin/perl -w
+
+package ippToPsps::StackBatch;
+
+use warnings;
+use strict;
+use File::Basename;
+use File::Temp qw(tempfile);
+use XML::LibXML;
+
+use ippToPsps::Batch;
+our @ISA = qw(ippToPsps::Batch);    # inherits from Batch class
+
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub existing {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::existing(
+            $_[1],  # camera
+            $_[2],  # gpc1Db
+            $_[3],  # batchId
+            $_[4],  # ippToPspsDb
+            $_[5],  # path
+            $_[6],  # verbose
+            $_[7]   # save_temps
+            );
+
+    bless $self, $class;
+    return $self;
+}
+
+
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::create(
+            "ST",   # type 
+            $_[2],  # camera
+            $_[3],  # gpc1Db
+            $_[4],  # ippToPspsDb
+            $_[5],  # path
+            $_[6],  # datastore
+            0,  # expId
+            "NULL",  # expName
+            $_[7],  # dvoDb
+            $_[8],  # dvoPath
+            "NULL", # distGroup
+            $_[9], # dontTarBall
+            $_[10], # verbose
+            $_[11]  # save_temps
+            );
+
+
+    $self->{_skyId} = $_[1]; 
+
+    bless $self, $class;
+    $self->process();
+    return $self;
+}
+
+#######################################################################################
+# 
+# Locates the relevant smf file for this exposure and writes the path to the file passed-in
+#
+#######################################################################################
+sub createInputFileList {
+    my ($self, $inputFile) = @_;
+
+    my ($nebPath, $numInputs);
+    if (!$self->{_gpc1Db}->getStackStageCmfs($self->{_skyId}, \$nebPath, \$numInputs)){
+
+        print "* No cmf files found for this skyId ($self->{_skyId})\n"; 
+        return 0;
+    }
+    # loop round all cmf files
+    my $i;
+    for ($i = 0; $i<$numInputs; $i++) {
+
+        # get real filename from neb 'key' TODO neater exit
+        my $fpaObjects = $self->{_ipprc}->filename("PSPHOT.STACK.OUTPUT",$nebPath, $i) or last;
+        my $cmf = $self->{_ipprc}->file_resolve($fpaObjects) or return 0;
+
+        print $inputFile $cmf . "\n";
+    }
+
+    return 1;
+}
+
+#######################################################################################
+#
+# - Reads results from processing (min/max obj ID etc)
+# - writes to Db
+# - writes manifest file
+#
+#######################################################################################
+sub parseResults {
+    my ($self, $resultsFilePath) = @_;
+
+    my($filename, $minObjId, $maxObjId, $totalDetections);
+
+    # read results of FITS creation from XML file
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file($resultsFilePath);
+    $filename = $doc->findvalue('//filename');
+    $minObjId = $doc->findvalue('//minObjID');
+    $maxObjId = $doc->findvalue('//maxObjID');
+    $totalDetections = $doc->findvalue('//totalDetections');
+
+    # write results to database
+    $self->{_ippToPspsDb}->setAsProcessed($self->{_batchId}, $self->{_expId}, $totalDetections, $minObjId, $maxObjId);
+
+    # write batch manifest
+    if (!$self->writeManifest($filename, $minObjId, $maxObjId)) {return 0;}
+
+    return 1;
+}
+1;
+
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeDetections.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeDetections.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeDetections.pl	(revision 29515)
@@ -0,0 +1,149 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use File::Basename;
+
+# local classes
+use ippToPsps::Gpc1Db;
+use ippToPsps::IppToPspsDb;
+use ippToPsps::Datastore;
+use ippToPsps::DetectionBatch;
+
+# globals
+my $camera = undef;
+my $dvoDb = undef;
+my $dvoPath = undef;
+my $verbose = undef;
+my $save_temps = undef; 
+my $output = undef;
+my $singleExpId = undef;
+my $lastExpId = undef;
+my $datastoreProduct = undef;
+my $force = undef;
+my $dontTarball = undef;
+my $datastore = undef;
+
+# get user args
+GetOptions(
+        'camera|c' => \$camera,
+        'output|o=s' => \$output,
+        'dvodb|d=s' => \$dvoDb,
+        'dvolocation|l=s' => \$dvoPath,
+        'expid|e=s' => \$singleExpId,
+        'lastexpid|x=s' => \$lastExpId,
+        'product|p=s' => \$datastoreProduct,
+        'verbose|v' => \$verbose,
+        'save_temps|t' => \$save_temps,
+        'force|f' => \$force,
+        'tarnzip|z' => \$dontTarball,
+        );
+
+my $quit = 0;
+print "\n*******************************************************************************\n";
+print "* \n";
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!defined $output) {
+    $quit=1;
+    print "* REQUIRED: need to provide an output path  -o <path>\n";
+}
+if (!defined $dvoDb) {
+    $quit=1;
+    print "* REQUIRED: need to provide a DVO Db        -d <dvoDb>\n";
+}
+if (!defined $dvoPath) {
+    $quit=1;
+    print "* REQUIRED: need to provide a DVO location  -l <dvoLocation>\n";
+}
+if (!defined $camera) {
+    $camera = "GPC1";
+    print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
+}
+if (!defined $singleExpId) {
+    $singleExpId = -1;
+    print "* OPTIONAL: a single exposure ID            -e <expID>           (default = none)\n";
+}
+if (!defined $lastExpId) {
+    $lastExpId = 0;
+    print "* OPTIONAL: process from last exp ID        -x <expID>           (default = $lastExpId)\n";
+}
+if (!defined $datastoreProduct) {
+
+    print "* OPTIONAL: datastore product               -p <product>         (default = none, i.e. data will not be published)\n";
+}
+if (!defined $verbose) {
+    $verbose = 0;
+    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+}
+if (!defined $save_temps) {
+    $save_temps = 0;
+    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+}
+if (!defined $force) {
+    $force = 0;
+    print "* OPTIONAL: force if already processed      -f                   (default = $force)\n";
+}
+if (!defined $dontTarball) {
+    $dontTarball = 0;
+    print "* OPTIONAL: don't tar and zip output        -z                   (default = $dontTarball)\n";
+}
+print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+
+my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
+my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);}
+
+my $exposures;
+
+my $query;
+# get single exposure
+if ($singleExpId != -1) {
+
+    if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {exit;}
+}
+# get all exposures in this DVO Db
+else {
+
+    if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures, $lastExpId)) {exit;}
+}
+
+my $exposure;
+foreach $exposure ( @{$exposures} ) {
+    my ($expId, $expName, $distGroup) = @{$exposure};
+
+
+    if ($ippToPspsDb->isExposureAlreadyPublished($expId)) {
+
+        if ($force) {print "* Already processed '$expId', but forcing....\n";}
+        else {
+
+            print "* Exposure ID '$expId' has already been processed and loaded to datastore, skipping\n";
+            next
+        };
+    }
+
+
+    my $detectionBatch = new ippToPsps::DetectionBatch(
+            $camera, 
+            $gpc1Db, 
+            $ippToPspsDb, 
+            $output, 
+            $datastore, 
+            $expId,
+            $expName,
+            $dvoDb,
+            $dvoPath,
+            $distGroup,
+            $dontTarball,
+            $verbose, 
+            $save_temps);
+
+}
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeInit.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeInit.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeInit.pl	(revision 29515)
@@ -0,0 +1,78 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+# local classes
+use ippToPsps::IppToPspsDb;
+use ippToPsps::Datastore;
+use ippToPsps::InitBatch;
+
+# globals
+my $camera = undef;
+my $verbose = undef;
+my $save_temps = undef; 
+my $output = undef;
+my $datastoreProduct = undef;
+my $dontTarball = undef;
+my $datastore = undef;
+
+# get user args
+GetOptions(
+        'camera|c' => \$camera,
+        'output|o=s' => \$output,
+        'product|p=s' => \$datastoreProduct,
+        'verbose|v' => \$verbose,
+        'save_temps|t' => \$save_temps,
+        'tarnzip|z' => \$dontTarball,
+        );
+
+my $quit = 0;
+print "\n*******************************************************************************\n";
+print "* \n";
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!defined $output) {
+    $quit=1;
+    print "* REQUIRED: need to provide an output path  -o <path>\n";
+}
+if (!defined $camera) {
+    $camera = "GPC1";
+    print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
+}
+if (!defined $datastoreProduct) {
+
+    print "* OPTIONAL: datastore product               -p <product>         (default = none, i.e. data will not be published)\n";
+}
+if (!defined $verbose) {
+    $verbose = 0;
+    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+}
+if (!defined $save_temps) {
+    $save_temps = 0;
+    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+}
+if (!defined $dontTarball) {
+    $dontTarball = 0;
+    print "* OPTIONAL: don't tar and zip output        -z                   (default = $dontTarball)\n";
+}
+print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);}
+
+my $initBatch = new ippToPsps::InitBatch(
+        $camera, 
+        $ippToPspsDb, 
+        $output, 
+        $datastore, 
+        $dontTarball,
+        $verbose, 
+        $save_temps);
+
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeStacks.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeStacks.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/makeStacks.pl	(revision 29515)
@@ -0,0 +1,105 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use File::Basename;
+
+# local classes
+use ippToPsps::Gpc1Db;
+use ippToPsps::IppToPspsDb;
+use ippToPsps::Datastore;
+use ippToPsps::StackBatch;
+
+# globals
+my $camera = undef;
+my $dvoDb = undef;
+my $dvoPath = undef;
+my $verbose = undef;
+my $save_temps = undef; 
+my $output = undef;
+my $datastoreProduct = undef;
+my $force = undef;
+my $dontTarball = undef;
+my $datastore = undef;
+
+# get user args
+GetOptions(
+        'camera|c' => \$camera,
+        'output|o=s' => \$output,
+        'dvodb|d=s' => \$dvoDb,
+        'dvolocation|l=s' => \$dvoPath,
+        'product|p=s' => \$datastoreProduct,
+        'verbose|v' => \$verbose,
+        'save_temps|t' => \$save_temps,
+        'force|f' => \$force,
+        'tarnzip|z' => \$dontTarball,
+        );
+
+my $quit = 0;
+print "\n*******************************************************************************\n";
+print "* \n";
+if (@ARGV) {
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
+if (!defined $output) {
+    $quit=1;
+    print "* REQUIRED: need to provide an output path  -o <path>\n";
+}
+if (!defined $dvoDb) {
+    $quit=1;
+    print "* REQUIRED: need to provide a DVO Db        -d <dvoDb>\n";
+}
+if (!defined $dvoPath) {
+    $quit=1;
+    print "* REQUIRED: need to provide a DVO location  -l <dvoLocation>\n";
+}
+if (!defined $camera) {
+    $camera = "GPC1";
+    print "* OPTIONAL: select a camera                 -c                   (default = $camera)\n";
+}
+if (!defined $datastoreProduct) {
+
+    print "* OPTIONAL: datastore product               -p <product>         (default = none, i.e. data will not be published)\n";
+}
+if (!defined $verbose) {
+    $verbose = 0;
+    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+}
+if (!defined $save_temps) {
+    $save_temps = 0;
+    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+}
+if (!defined $force) {
+    $force = 0;
+    print "* OPTIONAL: force if already processed      -f                   (default = $force)\n";
+}
+if (!defined $dontTarball) {
+    $dontTarball = 0;
+    print "* OPTIONAL: don't tar and zip output        -z                   (default = $dontTarball)\n";
+}
+print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+
+my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
+my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);}
+
+
+my $stackBatch = new ippToPsps::StackBatch(
+        11, # sky_id
+        $camera, 
+        $gpc1Db, 
+        $ippToPspsDb, 
+        $output, 
+        $datastore, 
+        $dvoDb,
+        $dvoPath,
+        $dontTarball,
+        $verbose, 
+        $save_temps);
+
Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/pspsSchema2xml.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/perl/pspsSchema2xml.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/pspsSchema2xml.pl	(revision 29515)
@@ -50,9 +50,13 @@
 
 
-my $output = new IO::File(">tables.xml");
-my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
-$writer->xmlDecl('UTF-8');
-#    $writer->doctype('manifest', "", "psps-manifest.dtd");
-$writer->startTag('tableDescriptions', "type" => "$type");
+my $tablesOutput = new IO::File(">tables.xml");
+my $tablesWriter = new XML::Writer(OUTPUT => $tablesOutput, DATA_MODE => 1, DATA_INDENT=>2);
+$tablesWriter->xmlDecl('UTF-8');
+$tablesWriter->startTag('tableDescriptions', "type" => "$type");
+
+my $mapOutput = new IO::File(">map.xml");
+my $mapWriter = new XML::Writer(OUTPUT => $mapOutput, DATA_MODE => 1, DATA_INDENT=>2);
+$mapWriter->xmlDecl('UTF-8');
+$mapWriter->startTag('tabledata', "type" => "$type");
 
 if ($type eq "init") {createInit();}
@@ -63,6 +67,10 @@
 
 # finish up XML
-$writer->endTag();
-$writer->end();
+$tablesWriter->endTag();
+$tablesWriter->end();
+
+# finish up XML
+$mapWriter->endTag();
+$mapWriter->end();
 
 print OUT "\n#endif";
@@ -164,7 +172,7 @@
     parseTable("StackMeta");
     parseTable("StackDetection");
-    parseTable("SkinnyObject");
-    parseTable("StackOrphan");
-    parseTable("ObjectCalColor");
+    parseTable("StackApFlx");
+    parseTable("StackModelFit");
+    parseTable("StackToImage");
 }
 
@@ -197,5 +205,8 @@
 
     print OUT "\ntypedef enum {\n";
-    $writer->startTag('table', "name" => $tableNameOut);
+    $tablesWriter->startTag('table', "name" => $tableNameOut);
+    $mapWriter->startTag('table', 
+            "name" => $tableNameOut,
+            "ippfitsextension" => "");
 
     open (SCHEMA, $path);
@@ -240,5 +251,6 @@
 
     if (!$found) {print "Could not find table '$tableName'\n";}
-    $writer->endTag();
+    $tablesWriter->endTag();
+    $mapWriter->endTag();
     print OUT "} ".$tableNameOut.";\n";
 
@@ -326,11 +338,14 @@
     print OUT "  ".uc($tableName)."_".uc($name)." = ".$colNum.",\n";
 
-    $writer->startTag('column',
+    $tablesWriter->startTag('column',
             "name" => $name,
             "type" => $type,
             "default" => $default,
             "comment" => $comment);
-
-    $writer->endTag();
+    $tablesWriter->endTag();
+
+    $mapWriter->comment(" <map pspsName=\"$name\" ippType=\"$type\" ippName=\"\" comment=\"$comment\"/>");
+
+#    $mapWriter->endTag();
 
     return $colNum;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.c	(revision 29515)
@@ -24,4 +24,15 @@
 extern int ippToPsps_batchDifference(IppToPsps *data);
 
+// Gets uncalibrated instrumental flux from magnitude
+__inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
+
+    *flux = powf(10.0, -0.4*magnitude) / exposureTime;
+    if (!isfinite(*flux) || *flux < 0.000001) return false;
+    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
+//      if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
+
+    return true;
+}
+
 // Destructor
 static void ippToPsps_Destructor(IppToPsps* this) {
@@ -55,5 +66,5 @@
     // save XML document for results
     if (this->resultsXmlDoc) {
-    
+
         xmlSaveFormatFileEnc(this->resultsPath, this->resultsXmlDoc, "UTF-8", 1);
 
@@ -151,5 +162,5 @@
 
     psMetadata *arguments = psMetadataAlloc();
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-expid", 0, "Exposure ID", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-id", 0, "ID", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-expname", 0, "Exposure name", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-survey", 0, "Survey type", NULL);
@@ -158,5 +169,5 @@
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-results", 0, "Path to results output", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-config", 0, "Path to config dir", NULL);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-batch", 0, "Product type: [det|stack|diff]", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-batch", 0, "Product type: [IN|P2|ST|TODO]", NULL);
 
     bool ret = true;
@@ -168,6 +179,6 @@
     else {
 
-        char *tmp = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-expid"));
-        if (tmp) this->expId = atoi(tmp);
+        char *tmp = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-id"));
+        if (tmp) this->id = atoi(tmp);
         //free(tmp); tmp = NULL;
         this->expName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-expname"));
@@ -181,8 +192,8 @@
         if (!tmp) this->batchType = BATCH_UNDEFINED;
         else if (strcmp(tmp, "test") == 0) this->batchType = BATCH_TEST;
-        else if (strcmp(tmp, "init") == 0) this->batchType = BATCH_INIT;
-        else if (strcmp(tmp, "det") == 0) this->batchType = BATCH_DETECTION;
-        else if (strcmp(tmp, "stack") == 0) this->batchType = BATCH_STACK;
-        else if (strcmp(tmp, "diff") == 0) this->batchType = BATCH_DIFFERENCE;
+        else if (strcmp(tmp, "IN") == 0) this->batchType = BATCH_INIT;
+        else if (strcmp(tmp, "P2") == 0) this->batchType = BATCH_DETECTION;
+        else if (strcmp(tmp, "ST") == 0) this->batchType = BATCH_STACK;
+        else if (strcmp(tmp, "TODO") == 0) this->batchType = BATCH_DIFFERENCE;
         else this->batchType = BATCH_UNDEFINED;
 
@@ -192,5 +203,5 @@
                 (this->batchType != BATCH_INIT && !this->fitsInPath) || 
                 (this->batchType != BATCH_INIT && !this->resultsPath) || 
-                (this->batchType != BATCH_INIT && this->expId == -1) ||
+                (this->batchType != BATCH_INIT && this->id == -1) ||
                 !this->fitsOutPath || 
                 !this->configsDir || 
@@ -211,4 +222,17 @@
 static int ippToPsps_run(IppToPsps *this) {
 
+    // any files?
+    if (this->batchType == BATCH_DETECTION ||
+            this->batchType == BATCH_STACK ||
+            this->batchType == BATCH_DIFFERENCE ) {
+
+        if (this->numOfInputFiles < 1) {
+
+            this->exitCode = PS_EXIT_DATA_ERROR;
+            psError(PS_ERR_UNKNOWN, false, "No input files for this batch");
+        }
+
+    }
+
     if(this->exitCode == PS_EXIT_SUCCESS) {
 
@@ -250,7 +274,18 @@
     psMemSetDeallocator(this, (psFreeFunc)ippToPsps_Destructor);
 
-    this->expId = -1;
+    // deal with DVO database
+    bool haveDvo = false;
+    for (int i=0;i<*argc; i++) {
+
+        if (strcmp(argv[i], "CATDIR") == 0) {haveDvo = true; break;}
+    }
+    if (haveDvo) this->dvoConfig = dvoConfigRead(argc, argv);
+    else this->dvoConfig = NULL;
+
+    // remaining args
+    this->id = -1;
     this->expName = NULL;
     this->surveyType = NULL;
+    this->surveyID = -1;
     this->fitsInPath = NULL;
     this->resultsPath = NULL;
@@ -261,10 +296,13 @@
     this->fitsOut = NULL;
     this->configsDir = NULL;
-    this->dvoConfig = NULL;
     this->config = NULL;
-    this->dvoConfig = dvoConfigRead(argc, argv);
     this->pmconfig = pmConfigRead(argc, argv, NULL);
     this->exitCode = PS_EXIT_SUCCESS;
     this->run = ippToPsps_run;
+
+    // sort out current time
+    time_t now = time(NULL);;
+    struct tm *ts = gmtime(&now); // TODO should be UTC
+    strftime(this->todaysDate, sizeof(this->todaysDate), "%Y-%m-%d", ts);
 
     if (!this->pmconfig) {
@@ -299,7 +337,7 @@
         psStringAppend(&this->configsDir, "/stack");
 
-
+    // create filename
     char outputName[100];
-    sprintf(outputName, "%08d.FITS", this->expId);
+    sprintf(outputName, "%08d.FITS", this->id);
     psStringAppend(&this->fitsOutPath, "/%s", outputName);
 
@@ -321,4 +359,12 @@
     }
 
+    // get survey ID using config object
+    if (this->batchType != BATCH_INIT && 
+            !ippToPspsConfig_getSurveyId(this->config, this->surveyType, &this->surveyID)) {
+        
+        this->exitCode = PS_EXIT_CONFIG_ERROR;
+        return this;
+    }
+
     // create XML document for results 
     if (this->resultsPath) {
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPsps.h	(revision 29515)
@@ -18,11 +18,12 @@
 #include <libxml/tree.h>
 
-#define MAXDETECT 30000 //TODO limit ok?
+#define MAXDETECT 60000 //TODO limit ok?
 
 typedef struct {
 
-    uint32_t expId;             // the exposure ID to be used
+    uint32_t id;                 // ID: might be expId for detections, skycell ID etc
     psString expName;           // the exposure name
     psString surveyType;        // the survey type, eg 3PI, MD01, STS, SSS
+    int8_t surveyID;            // survey ID
     uint8_t batchType;          // PSPS batch type
     psString fitsInPath;        // path to FITS input
@@ -37,4 +38,5 @@
     dvoConfig* dvoConfig;       // dvo database
     IppToPspsConfig* config;    // config structure
+    char todaysDate[12];        // today's date
     int exitCode;               // ps exit code
 
@@ -45,4 +47,10 @@
 IppToPsps *ippToPsps_Constructor(int *argc, char **argv);
 void ippToPsps_VersionPrint(void);
+bool ippToPsps_getFlux(
+        const float exposureTime, 
+        const float magnitude, 
+        float* flux, 
+        const float magnitudeErr, 
+        float* fluxErr);
 
 # endif // IPPTOPSPS_H 
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c	(revision 29515)
@@ -11,16 +11,4 @@
 #include "ippToPspsConfig.h"
 #include "ippToPspsDetEnums.h"
-#include "fitsio.h"
-
-// Gets uncalibrated instrumental flux from magnitude
-static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
-
-    *flux = powf(10.0, -0.4*magnitude) / exposureTime;
-    if (!isfinite(*flux) || *flux < 0.000001) return false;
-    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
-    //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
-         
-    return true;
-}
 
 /**
@@ -35,9 +23,6 @@
 int ippToPsps_batchDetection(IppToPsps *this) {
 
-    if (this->numOfInputFiles < 1) return PS_EXIT_DATA_ERROR;
-
     int status = 0;
     fitsfile *fitsIn;         
-    int nKeys;
 
     if (fits_open_file(&fitsIn, this->inputFiles[0], READONLY, &status)) {
@@ -48,4 +33,5 @@
 
     // get primary header and pull stuff out for later
+    int nKeys;
     fits_get_hdrspace(fitsIn, &nKeys, NULL, &status);
 
@@ -65,10 +51,8 @@
 
     // FrameMeta values
-    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
+    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
     fits_write_col(this->fitsOut, TSTRING, FRAMEMETA_FRAMENAME, 1, 1, 1, &this->expName, &status);
 
-    int8_t surveyID = -1;
-    if (!ippToPspsConfig_getSurveyId(this->config, this->surveyType, &surveyID)) {return PS_EXIT_DATA_ERROR;}
-    fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_SURVEYID, 1, 1, 1, &surveyID, &status);
+    fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_SURVEYID, 1, 1, 1, &this->surveyID, &status);
 
     int8_t filterID = -1;
@@ -97,11 +81,4 @@
 
     // stuff for detections table
-    time_t now;
-    struct tm *ts;
-    char timeStr[12];
-    now = time(NULL);
-    ts = gmtime(&now); // TODO should be UTC
-    strftime(timeStr, sizeof(timeStr), "%Y-%m-%d", ts);
-
     uint32_t s,d, invalidDvoRows, smfJumps, unmatched, totalDetections = 0;
 
@@ -128,6 +105,6 @@
 
         filterIDs[s] = filterID;
-        surveyIDs[s] = surveyID;
-        strcpy(assocDate[s], timeStr);
+        surveyIDs[s] = this->surveyID;
+        strcpy(assocDate[s], this->todaysDate);
     }
 
@@ -192,5 +169,5 @@
 
             // create unique int from 'frameID' (aka exposure ID) and ccd number
-            pspsImageId = (this->expId*100) + image->ccdnum;
+            pspsImageId = (this->id*100) + image->ccdnum;
 
             // now get DVO detections
@@ -206,5 +183,5 @@
             imageFlags = (uint64_t)image->flags;
             fits_write_col(this->fitsOut, TLONGLONG, IMAGEMETA_IMAGEID, 1, 1, 1, &pspsImageId, &status);
-            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
+            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
             fits_write_col(this->fitsOut, TSHORT, IMAGEMETA_CCDID, 1, 1, 1, &image->ccdnum, &status);
             fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &image->photcode, &status);
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchStack.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchStack.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchStack.c	(revision 29515)
@@ -9,4 +9,6 @@
 #include "ippToPsps.h"
 #include "ippToPspsConfig.h"
+#include "ippToPspsStackEnums.h"
+
 
 /**
@@ -15,13 +17,146 @@
 int ippToPsps_batchStack(IppToPsps *this) {
 
-//    char extensionName[20];
-    uint32_t skycell = 0; // TODO
+    int status = 0;
+    fitsfile *fitsIn;
 
-    for (uint16_t i = 0; i<this->numOfInputFiles; i++) {
+    if (fits_open_file(&fitsIn, this->inputFiles[0], READONLY, &status)) {
 
-        skycell = i; // TODO
+        fits_report_error(stderr, status);
+        return PS_EXIT_SYS_ERROR;
+    }
+
+    long removeList[MAXDETECT];
+    float instMag[MAXDETECT];
+    float floatnull = -999.0;
+    int anynull = 0;
+
+    // get primary header and pull stuff out for later
+    int nKeys = 0;
+    fits_get_hdrspace(fitsIn, &nKeys, NULL, &status);
+    float exposureTime; status=0; fits_read_key(fitsIn, TFLOAT, "EXPTIME", &exposureTime, NULL, &status);
+    char filterType[20]; status=0; fits_read_key(fitsIn, TSTRING, "FPA.FILTERID", filterType, NULL, &status);
 
 
+    char** assocDate = (char**)calloc(MAXDETECT, sizeof(char**));
+    for (uint32_t i=0; i<MAXDETECT;i++) assocDate[i] = (char*)calloc(20,sizeof(char));
+    int8_t filterIDs[MAXDETECT], surveyIDs[MAXDETECT];
+
+
+    // write StackMeta
+    ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "StackMeta", true);
+    fits_write_col(this->fitsOut, TLONG, STACKMETA_SKYCELLID, 1, 1, 1, &this->id, &status);
+
+    int8_t filterID = -1;
+    if (!ippToPspsConfig_getFilterId(this->config, filterType, &filterID)) {return PS_EXIT_DATA_ERROR;}
+    fits_write_col(this->fitsOut, TBYTE, STACKMETA_FILTERID, 1, 1, 1, &filterID, &status);
+
+    fits_write_col(this->fitsOut, TBYTE, STACKMETA_SURVEYID, 1, 1, 1, &this->surveyID, &status);
+
+
+    // psf detections
+    char extensionName[15];
+    sprintf(extensionName, "SkyChip.psf");
+    if (fits_movnam_hdu(fitsIn, BINARY_TBL, extensionName, 0, &status)) {
+        psError(PS_ERR_IO, false, "Can't move to extension: %s\n", extensionName);
+        
     }
+    else {
+   
+        // some stuff is the same for all detections so we can populate here
+        for (long s = 0; s<MAXDETECT; s++) {
+
+            filterIDs[s] = filterID;
+            surveyIDs[s] = this->surveyID;
+            strcpy(assocDate[s], this->todaysDate);
+        }
+
+
+        long nDet = 0;
+        if (fits_get_num_rows(fitsIn, &nDet, &status)) {
+            fits_report_error(stderr, status);
+        }
+
+        int instMagNum;
+        status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG", &instMagNum, &status);
+        if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG");
+        fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
+
+
+        printf("Looping through %ld psf detections\n", nDet);
+        float mag;
+        long unmatched = 0, totalDetections = 0, numOfDuplicates = 0, numInvalidFlux = 0, numDetectionsOut = 0;
+
+        for (long s = 0; s<nDet; s++) {
+
+            // TODO implement this match in DVO
+            if (1) {
+
+                mag = instMag[s];
+                if (!isfinite(mag) || mag < -998.0) {
+
+                    removeList[numOfDuplicates+numInvalidFlux] = s+1;
+                    numInvalidFlux++;
+                }
+
+                totalDetections++;
+            }
+            else {
+
+                unmatched++;
+                continue;
+            }
+        }
+
+        numDetectionsOut = totalDetections - numInvalidFlux;
+
+        if (numDetectionsOut > 0) {
+
+            ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, nDet, "StackDetection", false);
+            fits_write_col(this->fitsOut, TLONG, STACKDETECTION_SKYCELLID, 1, 1, 1, &this->id, &status);
+            fits_write_col(this->fitsOut, TBYTE, STACKDETECTION_FILTERID, 1, 1, nDet, filterIDs, &status);
+            fits_write_col(this->fitsOut, TBYTE, STACKDETECTION_SURVEYID, 1, 1, nDet, surveyIDs, &status);
+            fits_write_col(this->fitsOut, TSTRING, STACKDETECTION_ASSOCDATE, 1, 1, nDet, assocDate, &status);
+
+            if (numInvalidFlux) fits_delete_rowlist(this->fitsOut, removeList, numInvalidFlux, &status);
+
+        }
+        psLogMsg("ippToPsps", PS_LOG_INFO,
+                "+---------------+---------+----------+------------------+---------------+--------------+\n"
+                "|   Extension   | Rows in | Rows out | Missing from DVO | Duplicate IDs | Invalid Flux |\n"
+                "|  %12s |  %5ld  |   %5ld  |      %5ld       |    %5ld      |    %5ld     |\n",
+                extensionName, nDet, numDetectionsOut, unmatched, numOfDuplicates, numInvalidFlux);
+
+    }
+
+
+
+    // extended source
+    sprintf(extensionName, "SkyChip.xsrc");
+    if (fits_movnam_hdu(fitsIn, BINARY_TBL, extensionName, 0, &status)) {
+        psError(PS_ERR_IO, false, "Can't move to extension: %s\n", extensionName);
+
+    }
+    else {
+
+        long nDet = 0;
+        if (fits_get_num_rows(fitsIn, &nDet, &status)) {
+            fits_report_error(stderr, status);
+        }
+
+        printf("Looping through %ld extended source detections\n", nDet);
+        for (long s = 0; s<nDet; s++) {
+
+
+
+        }
+
+        ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, nDet, "StackApFlx", false);
+        fits_write_col(this->fitsOut, TBYTE, STACKAPFLX_FILTERID, 1, 1, nDet, filterIDs, &status);
+        fits_write_col(this->fitsOut, TBYTE, STACKAPFLX_SURVEYID, 1, 1, nDet, surveyIDs, &status);
+    }
+
+    status=0;
+    if (fits_close_file(fitsIn, &status)) fits_report_error(stderr, status);
+
 
     return PS_EXIT_SUCCESS;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchTest.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchTest.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchTest.c	(revision 29515)
@@ -13,14 +13,4 @@
 #include "fitsio.h"
 
-// Gets uncalibrated instrumental flux from magnitude
-static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
-
-    *flux = powf(10.0, -0.4*magnitude) / exposureTime;
-    if (!isfinite(*flux) || *flux < 0.000001) return false;
-    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
-    //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
-
-    return true;
-}
 
 /**
@@ -55,5 +45,5 @@
 
     // FrameMeta values
-//    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
+//    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->id, &status);
 
     int8_t filterID = -1;
@@ -187,5 +177,5 @@
 
             // create unique int from 'frameID' (aka exposure ID) and ccd number
-            pspsImageId = (this->expId*100) + pImage->ccdnum;
+            pspsImageId = (this->id*100) + pImage->ccdnum;
 
             // now get DVO detections
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.c	(revision 29515)
@@ -701,4 +701,29 @@
 }
 
+// gets metadata about a column 
+bool ippToPspsConfig_getFitsColumnMeta(
+        char* name,
+        int* colNum,
+        int* type,
+        long* repeat,
+        fitsfile *fitsIn ) {
+
+    int status = 0;
+    fits_get_colnum(fitsIn, CASESEN, name, colNum, &status);
+    if (status) {
+        psError(PS_ERR_IO, false, "Unable to read col '%s'", name);
+        return false;
+    }
+
+    status = 0;
+    fits_get_eqcoltype(fitsIn, *colNum, type, repeat, NULL, &status);
+    if (status) {
+        psError(PS_ERR_IO, false, "Unable to read type info for '%s'", name);
+        return false;
+    }
+
+    return true;
+}
+
 // populate with data from another FITS table into this one
 static bool ippToPspsConfig_populateTableFromFits(
@@ -730,5 +755,5 @@
     int readStatus = 0;
     int writeStatus = 0;
-   
+
     // first loop round all columns and get IPP col numbers for provided column names TODO only do once, first time in
     if(!fromHeader) {
@@ -737,7 +762,11 @@
 
             if (strlen(table->columns[i].ippName) < 1) continue;
-            readStatus = 0;
-            fits_get_colnum(fitsIn, CASESEN, table->columns[i].ippName, &table->columns[i].ippColNum, &readStatus);
-            if (readStatus) psError(PS_ERR_IO, false, "%d Unable to read col num for '%s' '%s' %d", i, table->columns[i].pspsName, table->columns[i].ippName, table->columns[i].ippColNum);
+
+            if (!ippToPspsConfig_getFitsColumnMeta(
+                    table->columns[i].ippName,
+                    &table->columns[i].ippColNum,
+                    &table->columns[i].ippType,
+                    &table->columns[i].ippRepeat,
+                    fitsIn)) {return false;}
         }
     }
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsConfig.h	(revision 29515)
@@ -20,4 +20,5 @@
     char ippName[100];
     int ippType;
+    long ippRepeat;
     char pspsName[100]; // TODO change to 'name'
     int pspsType;
Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsStackEnums.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsStackEnums.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsStackEnums.h	(revision 29515)
@@ -35,25 +35,27 @@
   STACKMETA_CRPIX1 = 30,
   STACKMETA_CRPIX2 = 31,
-  STACKMETA_PC001001 = 32,
-  STACKMETA_PC001002 = 33,
-  STACKMETA_PC002001 = 34,
-  STACKMETA_PC002002 = 35,
-  STACKMETA_POLYORDER = 36,
-  STACKMETA_PCA1X3Y0 = 37,
-  STACKMETA_PCA1X2Y1 = 38,
-  STACKMETA_PCA1X1Y2 = 39,
-  STACKMETA_PCA1X0Y3 = 40,
-  STACKMETA_PCA1X2Y0 = 41,
-  STACKMETA_PCA1X1Y1 = 42,
-  STACKMETA_PCA1X0Y2 = 43,
-  STACKMETA_PCA2X3Y0 = 44,
-  STACKMETA_PCA2X2Y1 = 45,
-  STACKMETA_PCA2X1Y2 = 46,
-  STACKMETA_PCA2X0Y3 = 47,
-  STACKMETA_PCA2X2Y0 = 48,
-  STACKMETA_PCA2X1Y1 = 49,
-  STACKMETA_PCA2X0Y2 = 50,
-  STACKMETA_CALIBMODNUM = 51,
-  STACKMETA_DATARELEASE = 52,
+  STACKMETA_CDELT1 = 32,
+  STACKMETA_CDELT2 = 33,
+  STACKMETA_PC001001 = 34,
+  STACKMETA_PC001002 = 35,
+  STACKMETA_PC002001 = 36,
+  STACKMETA_PC002002 = 37,
+  STACKMETA_POLYORDER = 38,
+  STACKMETA_PCA1X3Y0 = 39,
+  STACKMETA_PCA1X2Y1 = 40,
+  STACKMETA_PCA1X1Y2 = 41,
+  STACKMETA_PCA1X0Y3 = 42,
+  STACKMETA_PCA1X2Y0 = 43,
+  STACKMETA_PCA1X1Y1 = 44,
+  STACKMETA_PCA1X0Y2 = 45,
+  STACKMETA_PCA2X3Y0 = 46,
+  STACKMETA_PCA2X2Y1 = 47,
+  STACKMETA_PCA2X1Y2 = 48,
+  STACKMETA_PCA2X0Y3 = 49,
+  STACKMETA_PCA2X2Y0 = 50,
+  STACKMETA_PCA2X1Y1 = 51,
+  STACKMETA_PCA2X0Y2 = 52,
+  STACKMETA_CALIBMODNUM = 53,
+  STACKMETA_DATARELEASE = 54,
 } StackMeta;
 
@@ -107,64 +109,285 @@
 
 typedef enum {
-  SKINNYOBJECT_OBJID = 1,
-  SKINNYOBJECT_IPPOBJID = 2,
-  SKINNYOBJECT_PROJECTIONCELLID = 3,
-  SKINNYOBJECT_DATARELEASE = 4,
-} SkinnyObject;
-
-typedef enum {
-  STACKORPHAN_PARTITIONKEY = 1,
-  STACKORPHAN_STACKDETECTID = 2,
-  STACKORPHAN_IPPDETECTID = 3,
-  STACKORPHAN_FILTERID = 4,
-  STACKORPHAN_STACKTYPEID = 5,
-  STACKORPHAN_SURVEYID = 6,
-  STACKORPHAN_PRIMARYF = 7,
-  STACKORPHAN_STACKMETAID = 8,
-  STACKORPHAN_SKYCELLID = 9,
-  STACKORPHAN_STACKVER = 10,
-  STACKORPHAN_XPOS = 11,
-  STACKORPHAN_YPOS = 12,
-  STACKORPHAN_XPOSERR = 13,
-  STACKORPHAN_YPOSERR = 14,
-  STACKORPHAN_INSTFLUX = 15,
-  STACKORPHAN_INSTFLUXERR = 16,
-  STACKORPHAN_PEAKFLUX = 17,
-  STACKORPHAN_SKY = 18,
-  STACKORPHAN_SKYERR = 19,
-  STACKORPHAN_SGSEP = 20,
-  STACKORPHAN_PSFWIDMAJOR = 21,
-  STACKORPHAN_PSFWIDMINOR = 22,
-  STACKORPHAN_PSFTHETA = 23,
-  STACKORPHAN_PSFLIKELIHOOD = 24,
-  STACKORPHAN_PSFCF = 25,
-  STACKORPHAN_INFOFLAG = 26,
-  STACKORPHAN_NFRAMES = 27,
-  STACKORPHAN_WLSIGMA = 28,
-  STACKORPHAN_EPS1 = 29,
-  STACKORPHAN_EPS2 = 30,
-  STACKORPHAN_PSM11 = 31,
-  STACKORPHAN_PSM12 = 32,
-  STACKORPHAN_PSM21 = 33,
-  STACKORPHAN_PSM22 = 34,
-  STACKORPHAN_PSH11 = 35,
-  STACKORPHAN_PSH12 = 36,
-  STACKORPHAN_PSH21 = 37,
-  STACKORPHAN_PSH22 = 38,
-  STACKORPHAN_ACTIVEFLAG = 39,
-  STACKORPHAN_ASSOCDATE = 40,
-  STACKORPHAN_HISTORYMODNUM = 41,
-  STACKORPHAN_DATARELEASE = 42,
-} StackOrphan;
-
-typedef enum {
-  OBJECTCALCOLOR_OBJID = 1,
-  OBJECTCALCOLOR_IPPOBJID = 2,
-  OBJECTCALCOLOR_FILTERID = 3,
-  OBJECTCALCOLOR_CALCOLOR = 4,
-  OBJECTCALCOLOR_CALCOLORERR = 5,
-  OBJECTCALCOLOR_CALIBMODNUM = 6,
-  OBJECTCALCOLOR_DATARELEASE = 7,
-} ObjectCalColor;
+  STACKAPFLX_OBJID = 1,
+  STACKAPFLX_STACKDETECTID = 2,
+  STACKAPFLX_IPPOBJID = 3,
+  STACKAPFLX_IPPDETECTID = 4,
+  STACKAPFLX_FILTERID = 5,
+  STACKAPFLX_STACKTYPEID = 6,
+  STACKAPFLX_SURVEYID = 7,
+  STACKAPFLX_PRIMARYF = 8,
+  STACKAPFLX_STACKMETAID = 9,
+  STACKAPFLX_ISOPHOTMAG = 10,
+  STACKAPFLX_ISOPHOTMAGERR = 11,
+  STACKAPFLX_ISOPHOTMAJAXIS = 12,
+  STACKAPFLX_ISOPHOTMAJAXISERR = 13,
+  STACKAPFLX_ISOPHOTMINAXIS = 14,
+  STACKAPFLX_ISOPHOTMINAXISERR = 15,
+  STACKAPFLX_ISOPHOTMAJAXISGRAD = 16,
+  STACKAPFLX_ISOPHOTMINAXISGRAD = 17,
+  STACKAPFLX_ISOPHOTPA = 18,
+  STACKAPFLX_ISOPHOTPAERR = 19,
+  STACKAPFLX_ISOPHOTPAGRAD = 20,
+  STACKAPFLX_PETRADIUS = 21,
+  STACKAPFLX_PETRADIUSERR = 22,
+  STACKAPFLX_PETMAG = 23,
+  STACKAPFLX_PETMAGERR = 24,
+  STACKAPFLX_PETR50 = 25,
+  STACKAPFLX_PETR50ERR = 26,
+  STACKAPFLX_PETR90 = 27,
+  STACKAPFLX_PETR90ERR = 28,
+  STACKAPFLX_PETCF = 29,
+  STACKAPFLX_FLXR1 = 30,
+  STACKAPFLX_FLXR1ERR = 31,
+  STACKAPFLX_FLXR1VAR = 32,
+  STACKAPFLX_FLXR2 = 33,
+  STACKAPFLX_FLXR2ERR = 34,
+  STACKAPFLX_FLXR2VAR = 35,
+  STACKAPFLX_FLXR3 = 36,
+  STACKAPFLX_FLXR3ERR = 37,
+  STACKAPFLX_FLXR3VAR = 38,
+  STACKAPFLX_FLXR4 = 39,
+  STACKAPFLX_FLXR4ERR = 40,
+  STACKAPFLX_FLXR4VAR = 41,
+  STACKAPFLX_FLXR5 = 42,
+  STACKAPFLX_FLXR5ERR = 43,
+  STACKAPFLX_FLXR5VAR = 44,
+  STACKAPFLX_FLXR6 = 45,
+  STACKAPFLX_FLXR6ERR = 46,
+  STACKAPFLX_FLXR6VAR = 47,
+  STACKAPFLX_FLXR7 = 48,
+  STACKAPFLX_FLXR7ERR = 49,
+  STACKAPFLX_FLXR7VAR = 50,
+  STACKAPFLX_FLXR8 = 51,
+  STACKAPFLX_FLXR8ERR = 52,
+  STACKAPFLX_FLXR8VAR = 53,
+  STACKAPFLX_FLXR9 = 54,
+  STACKAPFLX_FLXR9ERR = 55,
+  STACKAPFLX_FLXR9VAR = 56,
+  STACKAPFLX_FLXR10 = 57,
+  STACKAPFLX_FLXR10ERR = 58,
+  STACKAPFLX_FLXR10VAR = 59,
+  STACKAPFLX_C1FLXR1 = 60,
+  STACKAPFLX_C1FLXR1ERR = 61,
+  STACKAPFLX_C1FLXR1VAR = 62,
+  STACKAPFLX_C1FLXR2 = 63,
+  STACKAPFLX_C1FLXR2ERR = 64,
+  STACKAPFLX_C1FLXR2VAR = 65,
+  STACKAPFLX_C1FLXR3 = 66,
+  STACKAPFLX_C1FLXR3ERR = 67,
+  STACKAPFLX_C1FLXR3VAR = 68,
+  STACKAPFLX_C1FLXR4 = 69,
+  STACKAPFLX_C1FLXR4ERR = 70,
+  STACKAPFLX_C1FLXR4VAR = 71,
+  STACKAPFLX_C1FLXR5 = 72,
+  STACKAPFLX_C1FLXR5ERR = 73,
+  STACKAPFLX_C1FLXR5VAR = 74,
+  STACKAPFLX_C1FLXR6 = 75,
+  STACKAPFLX_C1FLXR6ERR = 76,
+  STACKAPFLX_C1FLXR6VAR = 77,
+  STACKAPFLX_C1FLXR7 = 78,
+  STACKAPFLX_C1FLXR7ERR = 79,
+  STACKAPFLX_C1FLXR7VAR = 80,
+  STACKAPFLX_C1FLXR8 = 81,
+  STACKAPFLX_C1FLXR8ERR = 82,
+  STACKAPFLX_C1FLXR8VAR = 83,
+  STACKAPFLX_C1FLXR9 = 84,
+  STACKAPFLX_C1FLXR9ERR = 85,
+  STACKAPFLX_C1FLXR9VAR = 86,
+  STACKAPFLX_C1FLXR10 = 87,
+  STACKAPFLX_C1FLXR10ERR = 88,
+  STACKAPFLX_C1FLXR10VAR = 89,
+  STACKAPFLX_C2FLXR1 = 90,
+  STACKAPFLX_C2FLXR1ERR = 91,
+  STACKAPFLX_C2FLXR1VAR = 92,
+  STACKAPFLX_C2FLXR2 = 93,
+  STACKAPFLX_C2FLXR2ERR = 94,
+  STACKAPFLX_C2FLXR2VAR = 95,
+  STACKAPFLX_C2FLXR3 = 96,
+  STACKAPFLX_C2FLXR3ERR = 97,
+  STACKAPFLX_C2FLXR3VAR = 98,
+  STACKAPFLX_C2FLXR4 = 99,
+  STACKAPFLX_C2FLXR4ERR = 100,
+  STACKAPFLX_C2FLXR4VAR = 101,
+  STACKAPFLX_C2FLXR5 = 102,
+  STACKAPFLX_C2FLXR5ERR = 103,
+  STACKAPFLX_C2FLXR5VAR = 104,
+  STACKAPFLX_C2FLXR6 = 105,
+  STACKAPFLX_C2FLXR6ERR = 106,
+  STACKAPFLX_C2FLXR6VAR = 107,
+  STACKAPFLX_C2FLXR7 = 108,
+  STACKAPFLX_C2FLXR7ERR = 109,
+  STACKAPFLX_C2FLXR7VAR = 110,
+  STACKAPFLX_C2FLXR8 = 111,
+  STACKAPFLX_C2FLXR8ERR = 112,
+  STACKAPFLX_C2FLXR8VAR = 113,
+  STACKAPFLX_C2FLXR9 = 114,
+  STACKAPFLX_C2FLXR9ERR = 115,
+  STACKAPFLX_C2FLXR9VAR = 116,
+  STACKAPFLX_C2FLXR10 = 117,
+  STACKAPFLX_C2FLXR10ERR = 118,
+  STACKAPFLX_C2FLXR10VAR = 119,
+  STACKAPFLX_C3FLXR1 = 120,
+  STACKAPFLX_C3FLXR1ERR = 121,
+  STACKAPFLX_C3FLXR1VAR = 122,
+  STACKAPFLX_C3FLXR2 = 123,
+  STACKAPFLX_C3FLXR2ERR = 124,
+  STACKAPFLX_C3FLXR2VAR = 125,
+  STACKAPFLX_C3FLXR3 = 126,
+  STACKAPFLX_C3FLXR3ERR = 127,
+  STACKAPFLX_C3FLXR3VAR = 128,
+  STACKAPFLX_C3FLXR4 = 129,
+  STACKAPFLX_C3FLXR4ERR = 130,
+  STACKAPFLX_C3FLXR4VAR = 131,
+  STACKAPFLX_C3FLXR5 = 132,
+  STACKAPFLX_C3FLXR5ERR = 133,
+  STACKAPFLX_C3FLXR5VAR = 134,
+  STACKAPFLX_C3FLXR6 = 135,
+  STACKAPFLX_C3FLXR6ERR = 136,
+  STACKAPFLX_C3FLXR6VAR = 137,
+  STACKAPFLX_C3FLXR7 = 138,
+  STACKAPFLX_C3FLXR7ERR = 139,
+  STACKAPFLX_C3FLXR7VAR = 140,
+  STACKAPFLX_C3FLXR8 = 141,
+  STACKAPFLX_C3FLXR8ERR = 142,
+  STACKAPFLX_C3FLXR8VAR = 143,
+  STACKAPFLX_C3FLXR9 = 144,
+  STACKAPFLX_C3FLXR9ERR = 145,
+  STACKAPFLX_C3FLXR9VAR = 146,
+  STACKAPFLX_C3FLXR10 = 147,
+  STACKAPFLX_C3FLXR10ERR = 148,
+  STACKAPFLX_C3FLXR10VAR = 149,
+  STACKAPFLX_LOGC = 150,
+  STACKAPFLX_LOGA = 151,
+  STACKAPFLX_ACTIVEFLAG = 152,
+  STACKAPFLX_DATARELEASE = 153,
+} StackApFlx;
+
+typedef enum {
+  STACKMODELFIT_OBJID = 1,
+  STACKMODELFIT_STACKDETECTID = 2,
+  STACKMODELFIT_IPPOBJID = 3,
+  STACKMODELFIT_IPPDETECTID = 4,
+  STACKMODELFIT_FILTERID = 5,
+  STACKMODELFIT_STACKTYPEID = 6,
+  STACKMODELFIT_SURVEYID = 7,
+  STACKMODELFIT_PRIMARYF = 8,
+  STACKMODELFIT_STACKMETAID = 9,
+  STACKMODELFIT_DEVRADIUS = 10,
+  STACKMODELFIT_DEVRADIUSERR = 11,
+  STACKMODELFIT_DEVMAG = 12,
+  STACKMODELFIT_DEVMAGERR = 13,
+  STACKMODELFIT_DEVAB = 14,
+  STACKMODELFIT_DEVABERR = 15,
+  STACKMODELFIT_RADEVOFF = 16,
+  STACKMODELFIT_DECDEVOFF = 17,
+  STACKMODELFIT_RADEVOFFERR = 18,
+  STACKMODELFIT_DECDEVOFFERR = 19,
+  STACKMODELFIT_DEVCF = 20,
+  STACKMODELFIT_DEVLIKELIHOOD = 21,
+  STACKMODELFIT_DEVCOVAR11 = 22,
+  STACKMODELFIT_DEVCOVAR12 = 23,
+  STACKMODELFIT_DEVCOVAR13 = 24,
+  STACKMODELFIT_DEVCOVAR14 = 25,
+  STACKMODELFIT_DEVCOVAR15 = 26,
+  STACKMODELFIT_DEVCOVAR16 = 27,
+  STACKMODELFIT_DEVCOVAR22 = 28,
+  STACKMODELFIT_DEVCOVAR23 = 29,
+  STACKMODELFIT_DEVCOVAR24 = 30,
+  STACKMODELFIT_DEVCOVAR25 = 31,
+  STACKMODELFIT_DEVCOVAR26 = 32,
+  STACKMODELFIT_DEVCOVAR33 = 33,
+  STACKMODELFIT_DEVCOVAR34 = 34,
+  STACKMODELFIT_DEVCOVAR35 = 35,
+  STACKMODELFIT_DEVCOVAR36 = 36,
+  STACKMODELFIT_DEVCOVAR44 = 37,
+  STACKMODELFIT_DEVCOVAR45 = 38,
+  STACKMODELFIT_DEVCOVAR46 = 39,
+  STACKMODELFIT_DEVCOVAR55 = 40,
+  STACKMODELFIT_DEVCOVAR56 = 41,
+  STACKMODELFIT_DEVCOVAR66 = 42,
+  STACKMODELFIT_EXPRADIUS = 43,
+  STACKMODELFIT_EXPRADIUSERR = 44,
+  STACKMODELFIT_EXPMAG = 45,
+  STACKMODELFIT_EXPMAGERR = 46,
+  STACKMODELFIT_EXPAB = 47,
+  STACKMODELFIT_EXPABERR = 48,
+  STACKMODELFIT_RAEXPOFF = 49,
+  STACKMODELFIT_DECEXPOFF = 50,
+  STACKMODELFIT_RAEXPOFFERR = 51,
+  STACKMODELFIT_DECEXPOFFERR = 52,
+  STACKMODELFIT_EXPCF = 53,
+  STACKMODELFIT_EXPLIKELIHOOD = 54,
+  STACKMODELFIT_EXPCOVAR11 = 55,
+  STACKMODELFIT_EXPCOVAR12 = 56,
+  STACKMODELFIT_EXPCOVAR13 = 57,
+  STACKMODELFIT_EXPCOVAR14 = 58,
+  STACKMODELFIT_EXPCOVAR15 = 59,
+  STACKMODELFIT_EXPCOVAR16 = 60,
+  STACKMODELFIT_EXPCOVAR22 = 61,
+  STACKMODELFIT_EXPCOVAR23 = 62,
+  STACKMODELFIT_EXPCOVAR24 = 63,
+  STACKMODELFIT_EXPCOVAR25 = 64,
+  STACKMODELFIT_EXPCOVAR26 = 65,
+  STACKMODELFIT_EXPCOVAR33 = 66,
+  STACKMODELFIT_EXPCOVAR34 = 67,
+  STACKMODELFIT_EXPCOVAR35 = 68,
+  STACKMODELFIT_EXPCOVAR36 = 69,
+  STACKMODELFIT_EXPCOVAR44 = 70,
+  STACKMODELFIT_EXPCOVAR45 = 71,
+  STACKMODELFIT_EXPCOVAR46 = 72,
+  STACKMODELFIT_EXPCOVAR55 = 73,
+  STACKMODELFIT_EXPCOVAR56 = 74,
+  STACKMODELFIT_EXPCOVAR66 = 75,
+  STACKMODELFIT_SERRADIUS = 76,
+  STACKMODELFIT_SERRADIUSERR = 77,
+  STACKMODELFIT_SERMAG = 78,
+  STACKMODELFIT_SERMAGERR = 79,
+  STACKMODELFIT_SERAB = 80,
+  STACKMODELFIT_SERABERR = 81,
+  STACKMODELFIT_SERNU = 82,
+  STACKMODELFIT_SERNUERR = 83,
+  STACKMODELFIT_RASEROFF = 84,
+  STACKMODELFIT_DECSEROFF = 85,
+  STACKMODELFIT_RASEROFFERR = 86,
+  STACKMODELFIT_DECSEROFFERR = 87,
+  STACKMODELFIT_SERCF = 88,
+  STACKMODELFIT_SERLIKELIHOOD = 89,
+  STACKMODELFIT_SERSICCOVAR11 = 90,
+  STACKMODELFIT_SERSICCOVAR12 = 91,
+  STACKMODELFIT_SERSICCOVAR13 = 92,
+  STACKMODELFIT_SERSICCOVAR14 = 93,
+  STACKMODELFIT_SERSICCOVAR15 = 94,
+  STACKMODELFIT_SERSICCOVAR16 = 95,
+  STACKMODELFIT_SERSICCOVAR17 = 96,
+  STACKMODELFIT_SERSICCOVAR22 = 97,
+  STACKMODELFIT_SERSICCOVAR23 = 98,
+  STACKMODELFIT_SERSICCOVAR24 = 99,
+  STACKMODELFIT_SERSICCOVAR25 = 100,
+  STACKMODELFIT_SERSICCOVAR26 = 101,
+  STACKMODELFIT_SERSICCOVAR27 = 102,
+  STACKMODELFIT_SERSICCOVAR33 = 103,
+  STACKMODELFIT_SERSICCOVAR34 = 104,
+  STACKMODELFIT_SERSICCOVAR35 = 105,
+  STACKMODELFIT_SERSICCOVAR36 = 106,
+  STACKMODELFIT_SERSICCOVAR37 = 107,
+  STACKMODELFIT_SERSICCOVAR44 = 108,
+  STACKMODELFIT_SERSICCOVAR45 = 109,
+  STACKMODELFIT_SERSICCOVAR46 = 110,
+  STACKMODELFIT_SERSICCOVAR47 = 111,
+  STACKMODELFIT_SERSICCOVAR55 = 112,
+  STACKMODELFIT_SERSICCOVAR56 = 113,
+  STACKMODELFIT_SERSICCOVAR57 = 114,
+  STACKMODELFIT_SERSICCOVAR66 = 115,
+  STACKMODELFIT_SERSICCOVAR67 = 116,
+  STACKMODELFIT_SERSICCOVAR77 = 117,
+  STACKMODELFIT_ACTIVEFLAG = 118,
+  STACKMODELFIT_DATARELEASE = 119,
+} StackModelFit;
+
+typedef enum {
+  STACKTOIMAGE_STACKMETAID = 1,
+  STACKTOIMAGE_IMAGEID = 2,
+} StackToImage;
 
 #endif
Index: /branches/eam_branches/ipp-20100823/ippTools/configure.ac
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/configure.ac	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/configure.ac	(revision 29515)
@@ -1,5 +1,5 @@
 AC_PREREQ(2.61)
 
-AC_INIT([ipptools], [1.1.63], [ipp-support@ifa.hawaii.edu])
+AC_INIT([ipptools], [1.1.65], [ipp-support@ifa.hawaii.edu])
 AC_CONFIG_SRCDIR([autogen.sh])
 
@@ -18,5 +18,5 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.65]) 
 PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) 
 
Index: /branches/eam_branches/ipp-20100823/ippTools/share/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/Makefile.am	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/Makefile.am	(revision 29515)
@@ -61,4 +61,5 @@
 	chiptool_pendingimfile.sql \
 	chiptool_processedimfile.sql \
+	chiptool_revertcleanup.sql \
 	chiptool_revertprocessedimfile.sql \
 	chiptool_revertupdatedimfile.sql \
@@ -138,8 +139,10 @@
 	difftool_inputskyfile.sql \
 	difftool_listrun.sql \
+	difftool_listssrun.sql \
 	difftool_pendingcleanuprun.sql \
 	difftool_pendingcleanupskyfile.sql \
 	difftool_revertdiffskyfile_delete.sql \
 	difftool_revertdiffskyfile_updated.sql \
+	difftool_revertcleanup.sql \
 	difftool_setskyfiletoupdate.sql \
 	difftool_skyfile.sql \
@@ -274,4 +277,5 @@
 	pstamptool_revertreq.sql \
 	pstamptool_revertreq_deletejobs.sql \
+	pstamptool_stopdependentjob.sql \
 	pstamptool_updatejob.sql \
 	pxadmin_create_tables.sql \
@@ -356,4 +360,5 @@
 	warptool_revertwarped_delete.sql \
 	warptool_revertwarped_updated.sql \
+	warptool_revertcleanup.sql \
 	warptool_runstate.sql \
 	warptool_scmap.sql \
Index: /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id.sql	(revision 29515)
@@ -2,4 +2,6 @@
     camRun.*
 FROM camRun
+JOIN camProcessedExp
+    USING(cam_id)
 JOIN chipRun
     USING(chip_id)
@@ -10,5 +12,5 @@
            FROM addRun
            JOIN camRun USING(cam_id)
-           JOIN chipRun USING(chip_id)
+	   JOIN chipRun USING(chip_id)
           ) as foo
      ON exp_id = added_exp_id
@@ -17,4 +19,5 @@
 WHERE
     camRun.state = 'full'
+    AND camProcessedExp.quality = 0
     AND added_exp_id IS NULL
     -- addtool adds checks on exposure being added to the dvodb previously
Index: /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql	(revision 29515)
@@ -1,6 +1,7 @@
 SELECT camRun.* FROM camRun
+JOIN camProcessedExp USING(cam_id)
 JOIN chipRun USING(chip_id)
 JOIN rawExp USING(exp_id)
-WHERE camRun.state = 'full'
+WHERE camRun.state = 'full' and camProcessedExp.quality = 0
     AND exp_id NOT IN (SELECT exp_id
        FROM addRun
Index: /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_listrun.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_listrun.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_listrun.sql	(revision 29515)
@@ -29,3 +29,3 @@
 LEFT JOIN camProcessedExp USING(cam_id)
 LEFT JOIN magicDSRun
-    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place
+    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_processedimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_processedimfile.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_processedimfile.sql	(revision 29515)
@@ -42,3 +42,3 @@
     AND chipProcessedImfile.class_id = rawImfile.class_id
 LEFT JOIN magicDSRun
-    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place
+    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/chiptool_revertcleanup.sql	(revision 29515)
@@ -0,0 +1,7 @@
+UPDATE chipProcessedImfile
+    JOIN chipRun using(chip_id,exp_id)
+SET chipProcessedImfile.data_state = 'full', 
+    chipRun.state = '%s'
+WHERE
+    chipRun.state = '%s'
+    AND chipProcessedImfile.data_state = '%s'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listrun.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listrun.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listrun.sql	(revision 29515)
@@ -12,7 +12,5 @@
     diffRun.bothways,
     warp1,
-    stack1,
     warp2,
-    stack2,
     -- The following are only valid for warps
     -- XXX This needs to be more clever to handle diffs between stacks
@@ -27,4 +25,6 @@
     rawInput.exp_name AS exp_name_1,
     rawInput.exp_id AS exp_id_1,
+    rawInput.comment AS comment_1,
+    rawInput.dateobs AS dateobs_1,
     chipInput.chip_id AS chip_id_1,
     camInput.cam_id AS cam_id_1,
@@ -34,4 +34,6 @@
     rawTemplate.exp_name AS exp_name_2,
     rawTemplate.exp_id AS exp_id_2,
+    rawTemplate.comment AS comment_2,
+    rawTemplate.dateobs AS dateobs_2,
     chipTemplate.chip_id AS chip_id_2,
     camTemplate.cam_id AS cam_id_2,
Index: /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listssrun.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listssrun.sql	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/difftool_listssrun.sql	(revision 29515)
@@ -0,0 +1,20 @@
+SELECT DISTINCT
+    diffRun.diff_id,
+    diffRun.state,
+    diffRun.workdir,
+    diffRun.label,
+    diffRun.data_group,
+    diffRun.dist_group,
+    diffRun.note,
+    diffRun.tess_id
+FROM diffRun
+JOIN diffInputSkyfile USING(diff_id)
+JOIN stackRun AS stackInputRun
+    ON stackInputRun.stack_id = diffInputSkyfile.stack1
+JOIN stackSumSkyfile AS stackInput
+    ON stackInputRun.stack_id = stackInput.stack_id
+JOIN stackRun AS stackTemplateRun
+    ON stackTemplateRun.stack_id = diffInputSkyfile.stack2
+JOIN stackSumSkyfile AS stackTemplate
+    ON stackTemplateRun.stack_id = stackTemplate.stack_id
+WHERE diff_mode = 4
Index: /branches/eam_branches/ipp-20100823/ippTools/share/difftool_revertcleanup.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/difftool_revertcleanup.sql	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/difftool_revertcleanup.sql	(revision 29515)
@@ -0,0 +1,7 @@
+UPDATE diffSkyfile
+    JOIN diffRun using(diff_id)
+SET diffSkyfile.data_state = 'full', 
+    diffRun.state = '%s'
+WHERE
+    diffRun.state = '%s'
+    AND diffSkyfile.data_state = '%s'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/difftool_todiffskyfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/difftool_todiffskyfile.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/difftool_todiffskyfile.sql	(revision 29515)
@@ -13,5 +13,6 @@
     diffRun.bothways,
     diffRun.diff_mode,
-    diffSkyfile.path_base
+    diffSkyfile.path_base,
+    IFNULL(priority, 10000) AS priority
 FROM diffRun
 JOIN diffInputSkyfile USING(diff_id)
@@ -62,4 +63,5 @@
     ON diffInputSkyfile.diff_id = diffSkyfile.diff_id
     AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
+LEFT JOIN Label ON Label.label = diffRun.label
 WHERE
 -- Ready to be processed
@@ -70,4 +72,5 @@
     AND diffSkyfile.data_state = 'update')
     )
+    AND (Label.active OR Label.active IS NULL)
 -- Ensure input warps are available
     AND (diffInputSkyfile.warp1 IS NULL
@@ -93,4 +96,2 @@
     AND stackTemplateSkyfile.quality = 0))
 
-
-
Index: /branches/eam_branches/ipp-20100823/ippTools/share/disttool_pending_camera.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/disttool_pending_camera.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/disttool_pending_camera.sql	(revision 29515)
@@ -30,4 +30,4 @@
     AND distRun.stage = 'camera'
     AND distComponent.dist_id IS NULL
-    AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
+    AND (((clean OR (chipRun.magicked > 0)) AND (camRun.magicked > 0)) OR distRun.no_magic)
     AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_completed_runs.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_completed_runs.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_completed_runs.sql	(revision 29515)
@@ -1,4 +1,5 @@
 SELECT DISTINCT
     magic_ds_id,
+    magicked,
     re_place,
     label
@@ -7,7 +8,9 @@
 -- raw stage
 SELECT
-    magicDSRun.*
+    magicDSRun.*,
+    rawExp.magicked
     FROM magicDSRun
     JOIN rawImfile ON stage_id = rawImfile.exp_id
+    JOIN rawExp using(exp_id)
     LEFT JOIN magicDSFile
         ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
@@ -25,7 +28,9 @@
 -- chip stage
 SELECT
-    magicDSRun.*
+    magicDSRun.*,
+    chipRun.magicked
     FROM magicDSRun
-    JOIN chipProcessedImfile ON stage_id = chip_id
+    JOIN chipRun ON stage_id = chip_id
+    JOIN chipProcessedImfile USING(chip_id)
     LEFT JOIN magicDSFile
         ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
@@ -44,7 +49,9 @@
 -- camera stage
 SELECT
-    magicDSRun.*
+    magicDSRun.*,
+    camRun.magicked
     FROM magicDSRun
-    JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id
+    JOIN camRun ON stage_id = camRun.cam_id
+    JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id
     LEFT JOIN magicDSFile
         ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
@@ -61,7 +68,9 @@
 -- warp stage
 SELECT
-    magicDSRun.*
+    magicDSRun.*,
+    warpRun.magicked
     FROM magicDSRun
-    JOIN warpSkyfile on stage_id = warp_id
+    JOIN warpRun on stage_id = warp_id
+    JOIN warpSkyfile USING(warp_id)
     LEFT JOIN magicDSFile
         ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
@@ -81,10 +90,13 @@
 -- diff stage
 SELECT DISTINCT
-    magicDSRun.*
+    magicDSRun.*,
+    diffRun.magicked
     FROM magicDSRun
     JOIN magicRun USING (magic_id)
     JOIN magicInputSkyfile USING(magic_id)
+    JOIN diffRun
+        ON magicRun.diff_id = diffRun.diff_id
     JOIN diffSkyfile
-        ON magicRun.diff_id = diffSkyfile.diff_id
+        ON diffRun.diff_id = diffSkyfile.diff_id
         AND magicInputSkyfile.node = diffSkyfile.skycell_id
     LEFT JOIN magicDSFile
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql	(revision 29515)
@@ -7,5 +7,7 @@
     camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,
@@ -19,5 +21,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
@@ -29,4 +32,5 @@
 LEFT JOIN magicDSFile
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+LEFT JOIN Label ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
@@ -38,2 +42,3 @@
     AND camProcessedExp.quality = 0
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql	(revision 29515)
@@ -7,5 +7,7 @@
     camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,
@@ -19,5 +21,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
@@ -31,4 +34,6 @@
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     AND magicDSFile.component = chipProcessedImfile.class_id
+LEFT JOIN Label
+    ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
@@ -38,2 +43,3 @@
     AND chipProcessedImfile.quality = 0
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql	(revision 29515)
@@ -7,5 +7,7 @@
     rawExp.camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     magicRun.diff_id AS stage_id,
@@ -20,5 +22,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    IFNULL(Label.priority, 10000) AS priority
 FROM rawExp
 JOIN magicRun USING (exp_id)
@@ -33,4 +36,5 @@
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     AND magicDSFile.component = diffSkyfile.skycell_id
+LEFT JOIN Label ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
@@ -40,4 +44,5 @@
     AND diffSkyfile.quality = 0
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
 -- bothways diffSkyfiles
 UNION
@@ -49,5 +54,7 @@
     rawExp.camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri,
+    (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base,
     stage,
     magicRun.diff_id AS stage_id,
@@ -62,5 +69,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    IFNULL(Label.priority, 10000) AS priority
 FROM rawExp
 JOIN magicRun USING (exp_id)
@@ -75,4 +83,5 @@
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     AND magicDSFile.component = diffSkyfile.skycell_id
+LEFT JOIN Label ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
@@ -82,4 +91,5 @@
     AND diffSkyfile.quality = 0
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
 ) AS magicDSRun
 -- we need the following so this query is compatible with the other stages
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql	(revision 29515)
@@ -6,5 +6,7 @@
     rawExp.camera,
     magicMask.uri as streaks_uri,
+    magicMask.path_base as streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,
@@ -20,5 +22,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    10000 AS priority
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql	(revision 29515)
@@ -6,5 +6,7 @@
     camera,
     magicMask.uri as streaks_uri,
+    magicMask.path_base as streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,
@@ -19,5 +21,6 @@
     recoveryroot,
     re_place,
-    remove
+    remove,
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
@@ -29,4 +32,5 @@
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     AND magicDSFile.component = warpSkyfile.skycell_id
+LEFT JOIN Label ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
@@ -36,2 +40,3 @@
     AND warpSkyfile.quality = 0
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magictool_definebyquery_select.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magictool_definebyquery_select.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magictool_definebyquery_select.sql	(revision 29515)
@@ -15,7 +15,9 @@
     FROM diffRun
     JOIN diffInputSkyfile USING(diff_id)
+    JOIN diffSkyfile USING(diff_id, skycell_id)
     WHERE diffInputSkyfile.warp1 IS NOT NULL
         AND diffRun.exposure = 1
         AND diffRun.magicked = 0
+        AND diffSkyfile.quality = 0
     -- diff WHERE hook %s
     UNION
@@ -28,8 +30,10 @@
     FROM diffRun
     JOIN diffInputSkyfile USING(diff_id)
+    JOIN diffSkyfile USING(diff_id, skycell_id)
     WHERE diffInputSkyfile.warp2 IS NOT NULL
         AND diffRun.exposure = 1
         AND diffRun.bothways = 1
         AND diffRun.magicked = 0
+        AND diffSkyfile.quality = 0
     -- diff WHERE hook %s
     ) AS diffWarps
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magictool_revertnode.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magictool_revertnode.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magictool_revertnode.sql	(revision 29515)
@@ -3,2 +3,3 @@
 JOIN magicRun USING(magic_id) 
 WHERE magicNodeResult.fault != 0
+    AND magicRun.state = 'new'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql	(revision 29515)
@@ -3,4 +3,6 @@
     magicRun.workdir,
     rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.workdir AS raw_workdir,
     rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql	(revision 29515)
@@ -3,4 +3,6 @@
     magicRun.workdir,
     rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.workdir AS raw_workdir,
     rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
@@ -16,4 +18,5 @@
 WHERE
     magicRun.state = 'new'
+    AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
 ORDER BY
Index: /branches/eam_branches/ipp-20100823/ippTools/share/magictool_totree.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/magictool_totree.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/magictool_totree.sql	(revision 29515)
@@ -18,2 +18,3 @@
     AND magicTree.node IS NULL
     AND magicRun.fault = 0
+    AND (Label.active OR Label.active IS NULL)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertdependent.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertdependent.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertdependent.sql	(revision 29515)
@@ -3,4 +3,5 @@
     JOIN pstampRequest USING(req_id)
 SET pstampDependent.fault = 0
+-- fault count hook %s
 WHERE pstampDependent.state = 'new'
     AND (pstampDependent.fault > 0)
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertjob.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertjob.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_revertjob.sql	(revision 29515)
@@ -2,4 +2,5 @@
     JOIN pstampRequest USING(req_id)
 SET pstampJob.fault = 0
+ -- clear fault count clause goes here %s
 WHERE  pstampRequest.state = 'run'
     AND pstampJob.state = 'run'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_stopdependentjob.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_stopdependentjob.sql	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_stopdependentjob.sql	(revision 29515)
@@ -0,0 +1,9 @@
+-- stop jobs which have a faulted dependent
+UPDATE pstampDependent
+    JOIN pstampJob USING(dep_id)
+    JOIN pstampRequest USING(req_id)
+SET pstampJob.state = 'stop', 
+    pstampJob.fault = %d,
+    pstampDependent.fault = %d 
+WHERE pstampJob.state = 'run'
+    AND pstampDependent.state = 'new'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_updatejob.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_updatejob.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pstamptool_updatejob.sql	(revision 29515)
@@ -1,2 +1,4 @@
-UPDATE pstampJob LEFT JOIN pstampDependent USING(dep_id)
+UPDATE pstampJob 
+    JOIN pstampRequest USING(req_id) 
+    LEFT JOIN pstampDependent USING(dep_id)
 SET 
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pubtool_revert.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pubtool_revert.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pubtool_revert.sql	(revision 29515)
@@ -2,4 +2,5 @@
 USING publishDone, publishRun, publishClient
 WHERE publishDone.pub_id = publishRun.pub_id
+    AND publishRun.state = 'new'
     AND publishRun.client_id = publishClient.client_id
     AND publishClient.active = 1
Index: /branches/eam_branches/ipp-20100823/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/pxadmin_create_tables.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/pxadmin_create_tables.sql	(revision 29515)
@@ -1221,9 +1221,7 @@
 CREATE TABLE magicInputSkyfile (
         magic_id BIGINT,
-        diff_id BIGINT,
         node VARCHAR(64),
-        PRIMARY KEY(magic_id, diff_id, node),
-        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
-        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
+        PRIMARY KEY(magic_id, node),
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1253,4 +1251,5 @@
         magic_id BIGINT,
         uri VARCHAR(255),
+        path_base VARCHAR(255),
         streaks INT,
         fault SMALLINT,
@@ -1441,10 +1440,13 @@
         imagedb    VARCHAR(64),
         rlabel     VARCHAR(64),
+        need_magic TINYINT,
         outdir     VARCHAR(255),
-        need_magic TINYINT,
+        fault      SMALLINT,
+        fault_count INT,
         PRIMARY KEY(dep_id),
         KEY(state),
         KEY(stage),
-        KEY(stage_id)
+        KEY(stage_id),
+        KEY(fault)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1837,4 +1839,5 @@
     registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
     note VARCHAR(255),             -- note
+    magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied
     PRIMARY KEY(diff_phot_id),
     KEY(diff_id),
Index: /branches/eam_branches/ipp-20100823/ippTools/share/stacktool_tosum.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/stacktool_tosum.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/stacktool_tosum.sql	(revision 29515)
@@ -7,12 +7,15 @@
     stackRun.label,
     stackRun.state,
-    stackSumSkyfile.path_base
+    stackSumSkyfile.path_base,
+    IFNULL(Label.priority, 10000) AS priority
 FROM stackRun
 JOIN stackInputSkyfile USING(stack_id)
 JOIN warpRun USING(warp_id)
 LEFT JOIN stackSumSkyfile USING(stack_id)
+LEFT JOIN Label ON Label.label = stackRun.label
 WHERE
     ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL)
     OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0))
+    AND (Label.active OR Label.active IS NULL)
     -- WHERE hook %s
 GROUP BY stack_id
Index: /branches/eam_branches/ipp-20100823/ippTools/share/warptool_revertcleanup.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/warptool_revertcleanup.sql	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/warptool_revertcleanup.sql	(revision 29515)
@@ -0,0 +1,7 @@
+UPDATE warpSkyfile
+    JOIN warpRun using(warp_id)
+SET warpSkyfile.data_state = 'full', 
+    warpRun.state = '%s'
+WHERE
+    warpRun.state = '%s'
+    AND warpSkyfile.data_state = '%s'
Index: /branches/eam_branches/ipp-20100823/ippTools/share/warptool_scmap.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/warptool_scmap.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/warptool_scmap.sql	(revision 29515)
@@ -7,6 +7,10 @@
     chipRun.state,
     chipProcessedImfile.data_state,
+    chipProcessedImfile.fault AS chip_fault,
     chipProcessedImfile.magicked,
-    rawImfile.magicked AS raw_magicked
+    rawImfile.magicked AS raw_magicked,
+    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id,
+    IFNULL(magicDSRun.state, 0) AS dsRun_state,
+    IFNULL(magicDSFile.fault, 0) as dsFile_fault
 FROM warpRun
 JOIN warpSkyCellMap
@@ -26,4 +30,9 @@
     ON chipRun.exp_id = rawImfile.exp_id
     AND chipProcessedImfile.class_id = rawImfile.class_id
+LEFT JOIN magicDSRun
+    ON chipRun.chip_id = magicDSRun.stage_id AND magicDSRun.stage = 'chip'
+LEFT JOIN magicDSFile
+    ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id 
+        AND chipProcessedImfile.class_id = magicDSFile.component
 WHERE
     1
Index: /branches/eam_branches/ipp-20100823/ippTools/share/warptool_towarped.sql
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/share/warptool_towarped.sql	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/share/warptool_towarped.sql	(revision 29515)
@@ -97,12 +97,10 @@
         AND warpSkyfile.fault = 0
         AND camRun.state = 'full'
-        AND chipProcessedImfile.data_state = 'full'
-        -- if warpSkyfile was magicked previously require inputs to be magicked
-        -- this blocks processing until all the chip inputs have been destreaked
-        AND (warpSkyfile.magicked = 0 OR chipProcessedImfile.magicked >= 0)
         AND (Label.active OR Label.active IS NULL)
         -- where hook 2 %s
     GROUP BY warp_id, skycell_id
-    HAVING COUNT(warpSkyCellMap.class_id) = COUNT(chipProcessedImfile.class_id)
+    -- if warpSkyfile was magicked previously require inputs to be magicked
+    -- this blocks processing until all the chip inputs have been destreaked
+    HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full' and (chipRun.magicked = 0 OR chipProcessedImfile.magicked > 0), 1, 0))
     -- limit hook 2 %s
     )
Index: /branches/eam_branches/ipp-20100823/ippTools/src/addtool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/addtool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/addtool.c	(revision 29515)
@@ -493,9 +493,13 @@
 
     // since there is only one exp per 'new' set addRun.state = 'full'
-    if (!pxaddRunSetState(config, row->add_id, "full")) {
+    // but check to make sure there are no faults
+
+    if (!fault) {
+      if (!pxaddRunSetState(config, row->add_id, "full")) {
         psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id);
         psFree(row);
         psFree(pendingRow);
         return false;
+      }
     }
     psFree(row);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/bgtool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/bgtool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/bgtool.c	(revision 29515)
@@ -658,5 +658,5 @@
     }
 
-    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
+    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script,
                                     hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
                                     maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
@@ -1497,5 +1497,5 @@
     }
 
-    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
+    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script,
                                     hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
                                     maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
Index: /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.c	(revision 29515)
@@ -50,4 +50,5 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupimfileMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool runMode(pxConfig *config);
@@ -94,4 +95,5 @@
         MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN,       pendingcleanuprunMode);
         MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,    pendingcleanupimfileMode);
+        MODECASE(CHIPTOOL_MODE_REVERTCLEANUP,           revertcleanupMode);
         MODECASE(CHIPTOOL_MODE_DONECLEANUP,             donecleanupMode);
         MODECASE(CHIPTOOL_MODE_RUN,                     runMode);
@@ -859,9 +861,10 @@
     }
     psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql");
-    if (!query) {
+    if (!query_update) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
         return false;
     }
+
 
     if (where && psListLength(where->list)) {
@@ -886,4 +889,59 @@
     }
     psFree(query_update);
+
+    return true;
+}
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE");
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    char* newState = NULL;
+    if (!state) {
+        state = "error_cleaned";
+    }
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("chiptool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
 
     return true;
Index: /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/chiptool.h	(revision 29515)
@@ -42,4 +42,5 @@
     CHIPTOOL_MODE_PENDINGCLEANUPRUN,
     CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,
+    CHIPTOOL_MODE_REVERTCLEANUP,
     CHIPTOOL_MODE_DONECLEANUP,
     CHIPTOOL_MODE_RUN,
Index: /branches/eam_branches/ipp-20100823/ippTools/src/chiptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/chiptoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/chiptoolConfig.c	(revision 29515)
@@ -290,4 +290,12 @@
     psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-chip_id", 0,            "search by chip ID", 0);
+    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0,             "search by current state", NULL);
+    psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
+
     // -run
     psMetadata *runArgs = psMetadataAlloc();
@@ -371,4 +379,5 @@
     PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs);
     PXOPT_ADD_MODE("-donecleanup",          "show runs that have been cleaned",     CHIPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
+    PXOPT_ADD_MODE("-revertcleanup",        "show runs that have been cleaned",     CHIPTOOL_MODE_REVERTCLEANUP,        revertcleanupArgs);
     PXOPT_ADD_MODE("-run",                  "show runs",                            CHIPTOOL_MODE_RUN,                  runArgs);
     PXOPT_ADD_MODE("-tocleanedimfile",      "set imfile state to cleaned",          CHIPTOOL_MODE_TOCLEANEDIMFILE,      tocleanedimfileArgs);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/difftool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/difftool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/difftool.c	(revision 29515)
@@ -48,4 +48,5 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool updatediffskyfileMode(pxConfig *config);
@@ -60,4 +61,5 @@
 static bool tofullskyfileMode(pxConfig *config);
 static bool listrunMode(pxConfig *config);
+static bool listssrunMode(pxConfig *config);
 static bool setskyfiletoupdateMode(pxConfig *config);
 
@@ -99,4 +101,5 @@
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
+        MODECASE(DIFFTOOL_MODE_REVERTCLEANUP,         revertcleanupMode);
         MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
         MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
@@ -108,4 +111,5 @@
         MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
         MODECASE(DIFFTOOL_MODE_LISTRUN,               listrunMode);
+        MODECASE(DIFFTOOL_MODE_LISTSSRUN,             listssrunMode);
         MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE,    setskyfiletoupdateMode);
 
@@ -478,4 +482,6 @@
     }
     psFree(where);
+
+     psStringAppend(&query, "\nORDER by priority DESC, diff_id");
 
     // treat limit == 0 as "no limit"
@@ -1236,4 +1242,5 @@
     PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
 
     PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
@@ -1609,4 +1616,5 @@
     PXOPT_COPY_S64(config->args, selectWhere, "-warp_id", "inputWarpRun.warp_id", "==");
     PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
+    PXOPT_COPY_S64(config->args, selectWhere, "-template_exp_id", "templateRawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "==");
@@ -1647,5 +1655,6 @@
     PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_min",      "inputRawExp.sun_angle",      ">=");
     PXOPT_COPY_F32(config->args,   selectWhere, "-sun_angle_max",      "inputRawExp.sun_angle",      "<");
-    PXOPT_COPY_STR(config->args,   selectWhere, "-comment",            "inputRawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,   selectWhere, "-input_comment",      "inputRawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,   selectWhere, "-template_comment",   "templateRawExp.comment",     "LIKE");
 
     PXOPT_LOOKUP_BOOL(not_bothways, config->args, "-not-bothways", false);
@@ -1680,4 +1689,5 @@
 
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
@@ -1839,4 +1849,7 @@
     long numGood = 0;                   // Number of good rows added
     for (long i = 0; i < results->n; i++) {
+        if (limit && numGood >= limit) {
+            break;
+        }
         psMetadata *row = results->data[i]; // Result row from query
 
@@ -2535,4 +2548,61 @@
 
 
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-diff_id",    "diffSkyfile.diff_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label",     "diffRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group",   "diffRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    if (!state) {
+        state = "error_cleaned";
+    }
+    char *newState = NULL;
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    psString query = pxDataGet("difftool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+
+    psLogMsg("difftool", PS_LOG_INFO, "Reverted %d diffRuns and diffSkyfiles", numDeleted);
+
+    return true;
+}
 static bool donecleanupMode(pxConfig *config)
 {
@@ -2877,11 +2947,12 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
-    PXOPT_COPY_S64(config->args, where,  "-magicked", "diffRun.magicked", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE");
+    PXOPT_COPY_S64(config->args, where,  "-diff_id",    "diffRun.diff_id",  "==");
+    PXOPT_COPY_STR(config->args, where,  "-tess_id",    "diffRun.tess_id",  "LIKE");
+    PXOPT_COPY_S64(config->args, where,  "-magicked",   "diffRun.magicked", "==");
+    PXOPT_COPY_STR(config->args, where,  "-state",      "diffRun.state",    "==");
+    pxAddLabelSearchArgs (config, where, "-label",      "diffRun.label", "LIKE");
     pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE");
     pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE");
-
+    
     PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
     if (!template) {
@@ -2905,5 +2976,8 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
-
+    PXOPT_LOOKUP_S16(diff_mode, config->args, "-diff_mode", false, false);
+    if (diff_mode) {
+      PXOPT_COPY_S16(config->args, where, "-diff_mode", "diffRun.diff_mode", "==");
+    }
 
     psString where2 = NULL;
@@ -2983,4 +3057,121 @@
     return true;
 }
+static bool listssrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where,  "-diff_id",    "diffRun.diff_id",  "==");
+    PXOPT_COPY_S64(config->args, where,  "-stack_id",   "stackInput.stack_id",  "==");
+    PXOPT_COPY_S64(config->args, where,  "-template_stack_id",  "stackTemplate.stack_id",  "==");
+    PXOPT_COPY_STR(config->args, where,  "-tess_id",    "diffRun.tess_id",  "LIKE");
+    PXOPT_COPY_STR(config->args, where,  "-state",      "diffRun.state",    "==");
+    pxAddLabelSearchArgs (config, where, "-label",      "diffRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE");
+
+    // lookup these so we don't compare to zero if they are not supplied
+    PXOPT_LOOKUP_F64(mjd_obs_begin, config->args, "-mjd_obs_begin", false, false);
+    PXOPT_LOOKUP_F64(mjd_obs_end, config->args, "-mjd_obs_end", false, false);
+    
+    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
+
+    if (!template) {
+        if (mjd_obs_begin) {
+            PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackInput.mjd_obs",  ">=");
+        }
+        if (mjd_obs_end) {
+            PXOPT_COPY_F64(config->args, where, "-mjd_obs_end",   "stackInput.mjd_obs",  "<=");
+        }
+        PXOPT_COPY_STR(config->args, where, "-filter",        "stackInputRun.filter", "LIKE");
+    } else {
+        if (mjd_obs_begin) {
+            PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackTemplate.mjd_obs",  ">=");
+        }
+        if (mjd_obs_end) {
+            PXOPT_COPY_F64(config->args, where, "-mjd_obs_end",   "stackTemplate.mjd_obs",  "<=");
+        }
+        PXOPT_COPY_STR(config->args, where, "-filter",        "stackTemplateRun.filter", "LIKE");
+    }
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
+
+    psString where2 = NULL;
+    psString query = pxDataGet("difftool_listssrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    } else if (where2) {
+        psStringAppend(&query, " AND %s", where2);
+    } else if (!all) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+    psFree(where);
+
+    if (pstamp_order) {
+        if (template) {
+            psStringAppend(&query, " ORDER BY stackTemplate.stack_id, diff_id DESC");
+        } else {
+            psStringAppend(&query, " ORDER BY stackInput.stack_id, diff_id DESC");
+        }
+    }
+            
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
 // a very specfic function to queue a cleaned warpSkyfile to be updated
 static bool setskyfiletoupdateMode(pxConfig *config)
Index: /branches/eam_branches/ipp-20100823/ippTools/src/difftool.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/difftool.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/difftool.h	(revision 29515)
@@ -42,4 +42,5 @@
     DIFFTOOL_MODE_PENDINGCLEANUPRUN,
     DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE,
+    DIFFTOOL_MODE_REVERTCLEANUP,
     DIFFTOOL_MODE_DONECLEANUP,
     DIFFTOOL_MODE_UPDATEDIFFSKYFILE,
@@ -51,4 +52,5 @@
     DIFFTOOL_MODE_TOFULLSKYFILE,
     DIFFTOOL_MODE_LISTRUN,
+    DIFFTOOL_MODE_LISTSSRUN,
     DIFFTOOL_MODE_SETSKYFILETOUPDATE,
 } difftoolMode;
Index: /branches/eam_branches/ipp-20100823/ippTools/src/difftoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/difftoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/difftoolConfig.c	(revision 29515)
@@ -173,5 +173,5 @@
     psMetadata *listrunArgs = psMetadataAlloc();
     psMetadataAddS64(listrunArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
-    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID (LIKE comparison)", NULL);
     psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id",  0,         "search by warp_id", NULL);
     psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template",  0,        "apply exposure args to template of bothways diff", false);
@@ -184,4 +184,6 @@
     psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);
     psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-dist_group",  PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL);
+    psMetadataAddS16(listrunArgs,  PS_LIST_TAIL, "-diff_mode", 0,        "search for diff_mode", 0);
+    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-state",  0,           "search by state", NULL);
     pxmagicAddArguments(listrunArgs);
     pxspaceAddArguments(listrunArgs);
@@ -192,4 +194,29 @@
     psMetadataAddU64(listrunArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
+
+
+    // -listssrun
+    psMetadata *listssrunArgs = psMetadataAlloc();
+    psMetadataAddS64(listssrunArgs, PS_LIST_TAIL,  "-diff_id", 0,          "search by diff ID", 0);
+    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-tess_id",  0,         "search by tessellation ID (LIKE comparison)", NULL);
+    psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-stack_id",  0,        "search by input stack_id", 0);
+    psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-template_stack_id",0, "search by template stack_id", 0);
+    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-template",  0,        "apply stack selectors to template", false);
+    psMetadataAddF64(listssrunArgs, PS_LIST_TAIL,  "-mjd_obs_begin", 0,    "search by stack MJD-OBS (>=)", 0);
+    psMetadataAddF64(listssrunArgs, PS_LIST_TAIL,  "-mjd_obs_end", 0,      "search by stack MJD-OBS(<=)", 0);
+    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-filter", 0,           "search by stack filter", NULL);
+    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listssrunArgs,  PS_LIST_TAIL, "-dist_group",  PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL);
+    psMetadataAddS16(listssrunArgs,  PS_LIST_TAIL, "-diff_mode", 0,        "search for diff_mode", 0);
+    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-state",  0,           "search by state", NULL);
+    pxmagicAddArguments(listssrunArgs);
+    pxspaceAddArguments(listssrunArgs);
+
+    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-pstamp_order",  0,    "order results for postage stamp server", false);
+
+    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(listssrunArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(listssrunArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
 
 
@@ -226,4 +253,5 @@
     psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-comment", 0, "search by comment (LIKE)", NULL);
     psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_data_group", 0, "search by stack data_group", NULL);
     psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
     psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL);
@@ -247,4 +275,5 @@
     psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
     psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-not-bothways",  0, "only do the single-direction subtraction?", false);
+    psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL,  "-template_exp_id",  0,  "search by template exposure ID", 0);
     psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
     psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-distance", 0, "limit distance between input and template (deg)", NAN);
@@ -292,7 +321,9 @@
     psMetadataAddF64(definewarpwarpArgs,  PS_LIST_TAIL, "-posang_min",         0, "search by min rotator position angle", NAN);
     psMetadataAddF64(definewarpwarpArgs,  PS_LIST_TAIL, "-posang_max",         0, "search by max rotator position angle", NAN);
-    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_min",         0, "search by min solar angle", NAN);
-    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_max",         0, "search by max solar angle", NAN);
-    psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-comment",            0, "search by comment field (LIKE comparison)", NULL);
+    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_min",      0, "search by min solar angle", NAN);
+    psMetadataAddF32(definewarpwarpArgs,  PS_LIST_TAIL, "-sun_angle_max",      0, "search by max solar angle", NAN);
+    psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-input_comment",      0, "search by comment field for input exposure(LIKE comparison)", NULL);
+    psMetadataAddStr(definewarpwarpArgs,  PS_LIST_TAIL, "-template_comment",   0, "search by comment field for template exposure(LIKE comparison)", NULL);
+    psMetadataAddU64(definewarpwarpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
     // -definestackstack
@@ -362,4 +393,12 @@
     psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by difftool ID", 0);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+
+
     // -donecleanup
     psMetadata *donecleanupArgs = psMetadataAlloc();
@@ -427,5 +466,4 @@
     PXOPT_ADD_MODE("-advance",          "", DIFFTOOL_MODE_ADVANCE,           advanceArgs);
     PXOPT_ADD_MODE("-diffskyfile",      "", DIFFTOOL_MODE_DIFFSKYFILE,       diffskyfileArgs);
-    PXOPT_ADD_MODE("-listrun",          "", DIFFTOOL_MODE_LISTRUN,           listrunArgs);
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
@@ -433,6 +471,9 @@
     PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
     PXOPT_ADD_MODE("-definestackstack", "", DIFFTOOL_MODE_DEFINESTACKSTACK,  definestackstackArgs);
+    PXOPT_ADD_MODE("-listrun",          "list diff runs", DIFFTOOL_MODE_LISTRUN,           listrunArgs);
+    PXOPT_ADD_MODE("-listssrun",        "list stack-stack diff runs", DIFFTOOL_MODE_LISTSSRUN,           listssrunArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
+    PXOPT_ADD_MODE("-revertcleanup",           "revert cleanup runs with errors",     DIFFTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     DIFFTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-updatediffskyfile",     "update fault code for a diffskyfile",  DIFFTOOL_MODE_UPDATEDIFFSKYFILE,          updatediffskyfileArgs);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/magicdstool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/magicdstool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/magicdstool.c	(revision 29515)
@@ -637,4 +637,6 @@
     psFree(where);
 
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_ds_id");
+
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -933,4 +935,11 @@
     }
 
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -950,22 +959,29 @@
         return true;
     }
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
     for (long i = 0; i < psArrayLength(output); i++) {
         psMetadata *row = output->data[i];
 
         psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
+        psS64 magicked =  psMetadataLookupS64(NULL, row, "magicked");
+        if (!psDBTransaction(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
 
         // if re_place, set stageRun.magicked
         bool setmagicked = psMetadataLookupBool(NULL, row, "re_place");
-        if (setmagicked && !setRunMagicked(config, magic_ds_id)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
-                magic_ds_id);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
+        if (setmagicked) {
+            if (magicked <= 0) { 
+                if (!setRunMagicked(config, magic_ds_id)) {
+                    psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
+                        magic_ds_id);
+                    if (!psDBRollback(config->dbh)) {
+                        psError(PS_ERR_UNKNOWN, false, "database error");
+                    }
+                    return false;
+                }
+            } else {
+                fprintf(stderr, "run is already marked as destreaked for magic_ds_id %" PRId64 "\n", magic_ds_id);
             }
-            return false;
         }
 
@@ -980,8 +996,8 @@
             return false;
         }
-    }
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
     }
 
Index: /branches/eam_branches/ipp-20100823/ippTools/src/magictool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/magictool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/magictool.c	(revision 29515)
@@ -50,5 +50,5 @@
 static bool exposureMode(pxConfig *config);
 
-static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
+static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
 static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);
 
@@ -131,5 +131,5 @@
 
     psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
-    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "==");
 
@@ -394,9 +394,20 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
+
+    psString setString = NULL;
+    if (fault || clearfault) {
+        psStringAppend(&setString, ", fault = %d", fault);
+    }
+    if (note) {
+        psStringAppend(&setString, ", note = '%s'", note);
+    }
 
     if (state) {
         // set detRun.state to state
-        return setmagicRunState(config, magic_id, state);
+        return setmagicRunState(config, magic_id, state, setString);
     }
 
@@ -612,5 +623,5 @@
     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 
-    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
+    psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0");
 
     if (psListLength(where->list)) {
@@ -1171,5 +1182,6 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+//    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
     PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);
 
@@ -1184,5 +1196,6 @@
     if (!magicMaskInsert(config->dbh,
                          magic_id,
-                         uri,
+                         NULL,
+                         path_base,
                          streaks,
                          fault
@@ -1364,5 +1377,5 @@
 }
 
-static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
+static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -1380,7 +1393,13 @@
         return false;
     }
-
-    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) {
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state);
+    if (setString) {
+        psStringAppend(&query, setString);
+    }
+    psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);;
+
+//    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for magic_id %" PRId64, magic_id);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/magictoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/magictoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/magictoolConfig.c	(revision 29515)
@@ -79,5 +79,8 @@
     psMetadata *updaterunArgs = psMetadataAlloc();
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
+    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note",  0, "define note", NULL);
+    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault",  0, "set fault to zero", NULL);
 
     // -addinputskyfile
@@ -149,5 +152,5 @@
     psMetadata *addmaskArgs = psMetadataAlloc();
     psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
+    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL);
     psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0);
     psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.c	(revision 29515)
@@ -46,4 +46,5 @@
 static bool pendingjobMode(pxConfig *config);
 static bool updatejobMode(pxConfig *config);
+static bool stopdependentjobMode(pxConfig *config);
 static bool revertjobMode(pxConfig *config);
 static bool addprojectMode(pxConfig *config);
@@ -63,4 +64,7 @@
     break;
 
+// XXX make this a configurable parameter
+#define PSTAMP_MAX_JOB_FAULTS 5
+#define PSTAMP_MAX_DEP_FAULTS 5
 
 int main(int argc, char **argv)
@@ -89,4 +93,5 @@
         MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode);
         MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode);
+        MODECASE(PSTAMPTOOL_MODE_STOPDEPENDENTJOB, stopdependentjobMode);
         MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode);
         MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode);
@@ -715,5 +720,6 @@
             outputBase,
             options,
-            dep_id
+            dep_id,
+            0   // fault_count
             )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -877,7 +883,8 @@
     PXOPT_LOOKUP_S64(req_id,    config->args, "-req_id", false, false);
     PXOPT_LOOKUP_S64(dep_id,    config->args, "-dep_id", false, false);
-
-    if (!job_id && !req_id && !dep_id) {
-        psError(PS_ERR_UNKNOWN, true, "at least one of -job_id -req_id or -dep_id is required");
+    PXOPT_LOOKUP_S32(fault_count, config->args, "-fault_count",  false, false);
+
+    if (!job_id && !req_id && !dep_id && !fault_count) {
+        psError(PS_ERR_UNKNOWN, true, "at least one of -job_id -req_id -dep_id or -fault_count is required");
         return false;
     }
@@ -898,4 +905,6 @@
     PXOPT_COPY_S32(config->args, where, "-fault",  "pstampJob.fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state",  "pstampJob.state", "==");
+    PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampJob.fault_count", ">=");
+    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
 
     psString query = pxDataGet("pstamptool_updatejob.sql");
@@ -909,4 +918,5 @@
         psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault);
         c = ',';
+        psStringAppend(&query, ", pstampJob.fault_count = pstampJob.fault_count+ 1");
     }
     
@@ -925,4 +935,49 @@
     psU64 affected = psDBAffectedRows(config->dbh);
     psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs", affected);
+
+    return true;
+}
+// Terminate jobs which have dependents setting both the pstampDependent and pstampJob.fault
+static bool stopdependentjobMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S32(fault,  config->args, "-set_fault",  true, false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-fault",  "pstampDependent.fault", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
+    // if (fault_count) {
+        PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">=");
+    // }
+
+    // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself.
+
+    psString query = pxDataGet("pstamptool_stopdependentjob.sql");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampJobs and pstampDependents", affected);
 
     return true;
@@ -941,5 +996,10 @@
     pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
 
+    PXOPT_LOOKUP_BOOL(clear_fault_count, config->args, "-clear_fault_count", false);
+
     PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+    // XXX: we don't actually use -limit. It doesn't work for UPDATE
+    // it's an allowed arg because add_poll_args adds it
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_S16(fault, config->args, "-fault",  false, false);
@@ -947,5 +1007,5 @@
     // By default only revert faults < PSTAMP_FIRST_ERROR_CODE which are our "ipp exit codes"
     // codes larger than that are the pstamp request interface.
-    // Don't fault those unless -fault waa provided
+    // Don't fault those unless -fault was explicitly provided
     psString faultClause = psStringCopy("");
     if (!fault) {
@@ -966,5 +1026,15 @@
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, faultClause)) {
+    // don't keep reverting once the number of faults reaches some value unless 
+    // a parameter asking us to clear that count is provided
+    psString faultCountClause = NULL;
+    if (clear_fault_count) {
+        psStringAppend(&faultCountClause, "\n, pstampJob.fault_count = 0");
+    } else {
+        psStringAppend(&query, " AND pstampJob.fault_count < %d", PSTAMP_MAX_JOB_FAULTS);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, faultCountClause, faultClause)) {
+        psFree(faultCountClause);
         psFree(faultClause);
         psFree(query);
@@ -974,4 +1044,6 @@
 
     psFree(faultClause);
+    psFree(faultCountClause);
+    psFree(query);
 
     return true;
@@ -1178,5 +1250,6 @@
         need_magic,
         outdir,
-        0               // fault
+        0,              // fault
+        0               // fault_count
         )) {
         if (!psDBRollback(config->dbh)) {
@@ -1300,4 +1373,5 @@
         psStringAppend(&query, "%s fault = %d", needComma ? ", " : "", fault);
         needComma = true;
+        psStringAppend(&query, ", pstampDependent.fault_count = pstampDependent.fault_count + 1");
     }
     psStringAppend(&query, " WHERE dep_id = %" PRId64, dep_id);
@@ -1325,5 +1399,12 @@
     PXOPT_COPY_S64(config->args, where, "-fault", "pstampDependent.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
     pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "==");
+    PXOPT_LOOKUP_BOOL(clear_fault_count, config->args, "-clear_fault_count", false);
+
+    // XXX: we don't actually use -limit. It doesn't work for UPDATE
+    // it's an allowed arg because add_poll_args adds it
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     if (!psListLength(where->list)) {
@@ -1352,5 +1433,15 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    // don't keep reverting once the number of faults reaches some value unless 
+    // a parameter asking us to clear that count is provided
+    psString faultCountClause = NULL;
+    if (clear_fault_count) {
+        psStringAppend(&faultCountClause, "\n, pstampDependent.fault_count = 0");
+    } else {
+        psStringAppend(&faultCountClause, " ");
+        psStringAppend(&query, " AND pstampDependent.fault_count < %d", PSTAMP_MAX_DEP_FAULTS);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, faultCountClause)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/pstamptool.h	(revision 29515)
@@ -38,4 +38,5 @@
     PSTAMPTOOL_MODE_LISTJOB,
     PSTAMPTOOL_MODE_PENDINGJOB,
+    PSTAMPTOOL_MODE_STOPDEPENDENTJOB,
     PSTAMPTOOL_MODE_JOBRESULT,
     PSTAMPTOOL_MODE_UPDATEJOB,
Index: /branches/eam_branches/ipp-20100823/ippTools/src/pstamptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/pstamptoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/pstamptoolConfig.c	(revision 29515)
@@ -167,6 +167,20 @@
     psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,             "current state of jobs to update", 0);
     psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0,             "current value for job fault", 0);
+    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
+    psMetadataAddS32(updatejobArgs, PS_LIST_TAIL, "-fault_count", 0,       "select by fault_count (>=)", 0);
     psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0,            "new state", NULL);
     psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0,            "new result", 0);
+    psMetadataAddU64(updatejobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
+
+    // -stopdependentjob
+    psMetadata *stopdependentjobArgs = psMetadataAlloc();
+    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-req_id", 0,       "req_id of jobs to update", 0);
+    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-job_id", 0,       "job_id of jobs to update", 0);
+    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-dep_id", 0,       "dep_id of jobs to update", 0);
+    psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-fault", 0,        "current value for dependent fault", 0);
+    psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL,  "-fault_count", 0,   "select by fault_count (>=)", 0);
+    psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-set_fault", 0,    "new fault value for job and dependent (required)", 0);
+    psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
+    psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
 
     // -revertjob
@@ -177,6 +191,7 @@
     psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault",  0,     "fault to revert", 0);
     psMetadataAddStr(revertjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL);
+    psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-clear_fault_count", 0,       "clear job fault count", false);
     psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0,       "revert all faulted jobs", false);
-    psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0,      "limit result set to N items", 0);
+    psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);
 
     // -getdependent
@@ -201,7 +216,11 @@
     // -revertdependent
     psMetadata *revertdependentArgs = psMetadataAlloc();
-    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent", 0);
+    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "search by dep_id for dependent", 0);
+    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-job_id", 0, "search by job_ idfor dependent", 0);
+    psMetadataAddS64(revertdependentArgs, PS_LIST_TAIL, "-req_id", 0, "search by req_id for dependent", 0);
     psMetadataAddStr(revertdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK,   "define label for dependent ", NULL);
-    psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0,   "define fault", 0);
+    psMetadataAddS16(revertdependentArgs, PS_LIST_TAIL, "-fault",  0, "search by dependent fault", 0);
+    psMetadataAddBool(revertdependentArgs, PS_LIST_TAIL, "-clear_fault_count", 0,       "clear job fault count", false);
+    psMetadataAddU64(revertdependentArgs, PS_LIST_TAIL, "-limit", 0,  "limit result set to N items", 0);
 
     // -pendingdependent
@@ -262,4 +281,5 @@
     PXOPT_ADD_MODE("-pendingjob",      "", PSTAMPTOOL_MODE_PENDINGJOB,   pendingjobArgs);
     PXOPT_ADD_MODE("-updatejob",       "", PSTAMPTOOL_MODE_UPDATEJOB,    updatejobArgs);
+    PXOPT_ADD_MODE("-stopdependentjob", "", PSTAMPTOOL_MODE_STOPDEPENDENTJOB,  stopdependentjobArgs);
     PXOPT_ADD_MODE("-revertjob",       "", PSTAMPTOOL_MODE_REVERTJOB,    revertjobArgs);
 
Index: /branches/eam_branches/ipp-20100823/ippTools/src/pubtool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/pubtool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/pubtool.c	(revision 29515)
@@ -340,5 +340,5 @@
     }
 
-    if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) {
+    if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Database error");
         psFree(whereClause);
Index: /branches/eam_branches/ipp-20100823/ippTools/src/stacktool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/stacktool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/stacktool.c	(revision 29515)
@@ -901,4 +901,6 @@
     psFree(where);
 
+    psStringAppend(&query, "\nORDER by priority DESC, stack_id");
+
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -982,4 +984,5 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
+    PXOPT_LOOKUP_F64(mjd_obs, config->args, "-mjd_obs", false, false);
 
     PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
@@ -1047,4 +1050,5 @@
                                hostname,
                                good_frac,
+                               mjd_obs,
                                fault,
                                software_ver,
@@ -1094,5 +1098,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
     PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
@@ -1100,4 +1104,7 @@
     PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+    PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackSumSkyfile.mjd_obs", ">=");
+    PXOPT_COPY_F64(config->args, where, "-mjd_obs_end", "stackSumSkyfile.mjd_obs", "<=");
+
 
 //  The following three selectors are incompatible with the sql so omit them
Index: /branches/eam_branches/ipp-20100823/ippTools/src/stacktoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/stacktoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/stacktoolConfig.c	(revision 29515)
@@ -172,4 +172,5 @@
     psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
     psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
+    psMetadataAddF64(addsumskyfileArgs, PS_LIST_TAIL, "-mjd_obs",  0,            "define mjd_obs (average mjd_obs of inputs)", NAN);
     psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
@@ -185,5 +186,5 @@
     psMetadata *sumskyfileArgs= psMetadataAlloc();
     psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
-    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
+    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID (LIKE comparison)", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,         "search by skycell ID", 0);
 #ifdef notdef
@@ -196,4 +197,6 @@
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-data_group", 0,        "search by stackRun.data_group (LIKE comparison)", NULL);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter (LIKE comparison)", NULL);
+    psMetadataAddF64(sumskyfileArgs, PS_LIST_TAIL, "-mjd_obs_begin",  0,      "search by mjd_obs (average mjd_obs of inputs <=)", NAN);
+    psMetadataAddF64(sumskyfileArgs, PS_LIST_TAIL, "-mjd_obs_end",  0,      "search by mjd_obs (average mjd_obs of inputs>=)", NAN);
     psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
     psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -204,5 +207,5 @@
     psMetadata *sassskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(sassskyfileArgs, PS_LIST_TAIL, "-sass_id", 0,           "search by stack association ID", 0);
-    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
+    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID (LIKE comparison)", 0);
     psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-projection_cell", 0,         "search by projection cell", 0);
 
Index: /branches/eam_branches/ipp-20100823/ippTools/src/warptool.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/warptool.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/warptool.c	(revision 29515)
@@ -52,4 +52,5 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupwarpMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool tocleanedskyfileMode(pxConfig *config);
@@ -107,4 +108,5 @@
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN,  pendingcleanuprunMode);
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode);
+        MODECASE(WARPTOOL_MODE_REVERTCLEANUP,      revertcleanupMode);
         MODECASE(WARPTOOL_MODE_DONECLEANUP,        donecleanupMode);
         MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
@@ -1805,4 +1807,61 @@
 
 
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    if (!state) {
+        state = "error_cleaned";
+    }
+    char *newState = NULL;
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    psString query = pxDataGet("warptool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+
+    psLogMsg("warptool", PS_LOG_INFO, "Reverted %d warpRuns and warpSkyfiles", numDeleted);
+
+    return true;
+}
 static bool donecleanupMode(pxConfig *config)
 {
Index: /branches/eam_branches/ipp-20100823/ippTools/src/warptool.h
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/warptool.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/warptool.h	(revision 29515)
@@ -48,4 +48,5 @@
     WARPTOOL_MODE_PENDINGCLEANUPRUN,
     WARPTOOL_MODE_PENDINGCLEANUPSKYFILE,
+    WARPTOOL_MODE_REVERTCLEANUP,
     WARPTOOL_MODE_DONECLEANUP,
     WARPTOOL_MODE_TOCLEANEDSKYFILE,
Index: /branches/eam_branches/ipp-20100823/ippTools/src/warptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100823/ippTools/src/warptoolConfig.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/ippTools/src/warptoolConfig.c	(revision 29515)
@@ -362,4 +362,11 @@
     psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+
     // -donecleanup
     psMetadata *donecleanupArgs = psMetadataAlloc();
@@ -451,4 +458,5 @@
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
+    PXOPT_ADD_MODE("-revertcleanup",         "revert cleanup runs with errors",     WARPTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     WARPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs);
Index: /branches/eam_branches/ipp-20100823/magic/remove/src/streaksio.c
===================================================================
--- /branches/eam_branches/ipp-20100823/magic/remove/src/streaksio.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/magic/remove/src/streaksio.c	(revision 29515)
@@ -1266,40 +1266,7 @@
         return;
     }
-    if (sfiles->inMask && sfiles->inMask->header) {
-        if (!pmConfigMaskReadHeader(sfiles->config, sfiles->inMask->header)) {
-            streaksExit("failed to read mask values from file", PS_EXIT_CONFIG_ERROR);
-        }
-    }
-
-    bool status;
-    psMetadata *masks = psMetadataLookupMetadata(&status, sfiles->config->recipes, "MASKS");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-    sfiles->maskStreak = psMetadataLookupU32(&status, masks, "STREAK");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-
-    // optionally setting pixels with any mask bits execpt CONV.POOR to NAN
-    // check old name first
-    psU32 convPoor = (double) psMetadataLookupU32(&status, masks, "POOR.WARP");
-    if (!status) {
-        convPoor = (double) psMetadataLookupU32(&status, masks, "CONV.POOR");
-        if (!status) {
-            psError(PM_ERR_CONFIG, false, "failed to lookup mask value for CONV.POOR in recipes\n");
-            streaksExit("", PS_EXIT_CONFIG_ERROR);
-        }
-    }
-    // preserve pixels that are only suspect
-    psU32 suspect = (double) psMetadataLookupU32(&status, masks, "SUSPECT");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for SUSPECT in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-
-    sfiles->maskMask = ~(convPoor | suspect);
+    if (!pmCensorGetMasks(sfiles->config, &sfiles->maskMask, &sfiles->maskStreak)) {
+        streaksExit("failed to find mask values", PS_EXIT_CONFIG_ERROR);
+    }
 }
 
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am	(revision 29515)
@@ -20,4 +20,5 @@
         pstamp_get_image_job.pl \
 	psmkreq \
+	pstampstopfaulted \
 	pstamp_checkdependent.pl \
 	request_finish.pl \
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_create
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_create	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_create	(revision 29515)
@@ -307,5 +307,12 @@
     while (<$in>) {
 	chomp;
+#	print STDERR "$line_num $#keywords $_\n";
+	if ($_ =~ /^\s*$/) {
+	    next;
+	}
 	if ($line_num == 0) {
+	    if ($_ !~ /EXTVER/) {
+		next;
+	    }
 	    # Parse header information keywords
 	    $_ =~ s/#//g;
@@ -319,5 +326,5 @@
 	    my @values = split /\s+/;
 	    if ($#values != $#keywords) {
-		die "Number of header columns in input does not equal expected number of header words";
+		die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
 	    }
 	    for (my $i = 0; $i <= $#values; $i++) {
@@ -326,4 +333,7 @@
 	}
 	elsif ($line_num == 2) {
+	    if ($_ !~ /ROWNUM/) {
+		next;
+	    }
 	    # Parse table information keywords, dumping old keywords
 	    $_ =~ s/#//g;
@@ -338,5 +348,5 @@
 		my @values = split /\s+/;
 		if ($#values != $#keywords) {
-		    die "Number of header columns in input does not equal expected number of header words";
+		    die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
 		}
 		for (my $i = 0; $i <= $#values; $i++) {
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_read
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_read	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/detect_query_read	(revision 29515)
@@ -36,5 +36,6 @@
 
 my $regtool = can_run('regtool') or (die "Can't find regtool");
-my $camtool = can_run('camtool') or (die "Can't find regtool");
+my $camtool = can_run('camtool') or (die "Can't find camtool");
+my $difftool = can_run('difftool') or (die "Can't find difftool");
 
 
@@ -164,5 +165,5 @@
     }
     elsif ($status == 219) {
-	@{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1);
+	@{ $colData{$col->{name}} } = map { "Not_Set" } (0 .. $numRows - 1);
 	$status = 0;
     }
@@ -173,12 +174,12 @@
 # Simple stuff first.
     my %known_filters = ();
-    if ($colData{STAGE}[$i] eq "Not Set") {
+    if ($colData{STAGE}[$i] eq "Not_Set") {
 	$colData{STAGE}[$i] = 'diff';
     }
-    if ($colData{OBSCODE}[$i] eq "Not Set") {
+    if ($colData{OBSCODE}[$i] eq "Not_Set") {
 	$colData{OBSCODE}[$i] = 566;
     }
 # Define filter and MJD from FPA_ID
-    if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+    if (($colData{FILTER}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
 	if (exists($known_filters{$colData{FPA_ID}[$i]})) {
 	    $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
@@ -210,5 +211,6 @@
 	}
     }
-    if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+    # Calculate a MJD
+    if (($colData{'MJD-OBS'}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
 	# HACK!
 	my $mjd = $colData{FPA_ID}[$i];
@@ -217,6 +219,7 @@
 	$colData{'MJD-OBS'}[$i] = $mjd;
     }
-    if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&&
-					      ($colData{'MJD-OBS'}[$i] ne "Not Set"))) {
+    # Calculate an exp_id
+    if (($colData{FPA_ID}[$i] eq "Not_Set")&&(($colData{'FILTER'}[$i] ne "Not_Set")&&
+					      ($colData{'MJD-OBS'}[$i] ne "Not_Set"))) {
 	my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
 	my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
@@ -224,9 +227,27 @@
 	my $dec = $colData{'DEC1_DEG'}[$i];
 	my $filter = $colData{'FILTER'}[$i] . ".00000";
-	my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
-	$cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
-	$cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+	my $cmd;
+	if ($colData{STAGE}[$i] eq 'SSdiff') {
+	    $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
+	    $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+	    $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 4";
+	}
+	elsif ($colData{STAGE}[$i] eq 'WSdiff') {
+	    $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
+	    $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+	    $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 2";
+	}
+	elsif ($colData{STAGE}[$i] eq 'diff') {
+	    $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
+	    $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+	    $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+	}
+	else {
+	    $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
+	    $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+	    $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+	}
 	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run(command => $cmd, verbose => 0);
+	    run(command => $cmd, verbose => 1);
 	unless ($success) {
 	    # This is a problem, because I'm not sure how we handle a failure to read something.
@@ -243,8 +264,22 @@
 	    my ($key,$type,$value) = @line;
 #		print "$entry => $key / $type / $value \n";
-	    if ($key =~ /exp_name/i) {
-		$colData{FPA_ID}[$i] = $value;
-	    }
-	}
+	    if ($colData{STAGE}[$i] =~ /diff/) {
+		if ($key =~ /diff_id/i) {
+		    $colData{FPA_ID}[$i] = $value;
+		}
+	    }
+	    else {
+		if ($key =~ /exp_name/i) {
+		    $colData{FPA_ID}[$i] = $value;
+		}
+	    }
+	}
+    }
+    # Correct diff modes
+    if ($colData{STAGE}[$i] eq 'WSdiff') {
+	$colData{STAGE}[$i] = 'diff';
+    }
+    elsif ($colData{STAGE}[$i] eq 'SSdiff') {
+	$colData{STAGE}[$i] = 'diff';
     }
 }
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl	(revision 29515)
@@ -91,8 +91,8 @@
     print "Reading wisdom file $wisdom_file instead of parsing...\n";
     open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
-    my $i = 0;
     while(<WISDOM>) {
 	chomp;
 	my ($fpa_id,@key_values) = split /\s+/;
+	my $index = $#{ $query{$fpa_id}{ROWNUM} } + 1;
 	while ($#key_values > -1) {
 	    my $key = shift(@key_values);
@@ -102,7 +102,6 @@
 		$val = 0;
 	    }
-	    $query{$fpa_id}{$key}[$i] = $val;
-	}
-	$i++;
+	    $query{$fpa_id}{$key}[$index] = $val;
+	}
     }
     close(WISDOM);
@@ -150,52 +149,32 @@
 	my $filter;
 	my $mjd;
-	# Determine the query style for this fpa_id
-	if ($fpa_id =~ /o.*g.*o/) {
-	    $query_style = 'byexp';
-	}
-	elsif ($fpa_id =~ /\d+/) {
-	    $query_style = 'byid';
-	}
-	else {
-	    exit_with_failure(21,"Parse error in request file");
-	}
 	# Confirm that we only have one stage/filter/mjd
-	for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
-	    $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
-	    $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
-	    $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
-	}
-	if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
-	    $stage = (keys(%{ $temp_hash{STAGE} }))[0];
-	}
-	else {
-	    exit_with_failure(21,"Too many STAGEs specified");
-	}
-	if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
-	    $filter = (keys(%{ $temp_hash{FILTER} }))[0];
-	}
-	else {
-	    exit_with_failure(21,"Too many FILTERs specified");
-	}
-	if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
-	    $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
-	}
-	else {
-	    exit_with_failure(21,"Too many MJD-OBS specified");
-	}
-	# Set common request components
-	my $option_mask |= 1;
-	$option_mask |= $PSTAMP_SELECT_IMAGE;
-	$option_mask |= $PSTAMP_SELECT_MASK;
-	$option_mask |= $PSTAMP_SELECT_VARIANCE;
-	$option_mask |= $PSTAMP_SELECT_PSF;
-	my $need_magic = 1;
-	if ($stage eq 'stack') {
-	    $need_magic = 0;
-	}
-	my $mjd_min = $mjd;
-	my $mjd_max = $mjd + 1;
-	
-	# Construct a row list. 
+	if ($fpa_id ne 'Not_Set') {   # We only need to check things that aren't the known odd case.
+	    for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+		$temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
+		$temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
+		$temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
+	    }
+	    if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
+		$stage = (keys(%{ $temp_hash{STAGE} }))[0];
+	    }
+	    else {
+		exit_with_failure(21,"Too many STAGEs specified");
+	    }
+	    if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
+		$filter = (keys(%{ $temp_hash{FILTER} }))[0];
+	    }
+	    else {
+		exit_with_failure(21,"Too many FILTERs specified");
+	    }
+	    if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
+		$mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
+	    }
+	    else {
+		exit_with_failure(21,"Too many MJD-OBS specified");
+	    }
+	}
+
+	# Set up a rowList with default values
 	my @rowList;
 	for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
@@ -223,6 +202,32 @@
 	    $query{$fpa_id}{FAULT}[$i] = 'no_fault';
 	    $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
-
-	}
+	}
+
+	# Determine the query style for this fpa_id
+	if ($fpa_id =~ /o.*g.*o/) {
+	    $query_style = 'byexp';
+	}
+	elsif ($fpa_id =~ /\d+/) {
+	    $query_style = 'byid';
+	}
+	elsif ($fpa_id eq 'Not_Set') {
+	    next;
+	}
+	else {
+	    exit_with_failure(21,"Parse error in request file");
+	}
+	# Set common request components
+	my $option_mask |= 1;
+	$option_mask |= $PSTAMP_SELECT_IMAGE;
+	$option_mask |= $PSTAMP_SELECT_MASK;
+	$option_mask |= $PSTAMP_SELECT_VARIANCE;
+	$option_mask |= $PSTAMP_SELECT_PSF;
+	my $need_magic = 1;
+	if ($stage eq 'stack') {
+	    $need_magic = 0;
+	}
+	my $mjd_min = $mjd;
+	my $mjd_max = $mjd + 1;
+	
 	
 	# Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
@@ -321,5 +326,5 @@
 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
 foreach my $fpa_id (keys %query) {
-    for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+    for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
 	print WISDOM "$fpa_id\t";
 	foreach my $key (keys %{ $query{$fpa_id} }) {
@@ -639,5 +644,5 @@
 	$columns = [
 	    # matching rownum from detectability original request
-	    { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
+	    { name => 'ROWNUM',   type => '20A', writetype => TULONG }, 
 	    # any errors that occurred during processing
 	    { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
@@ -690,5 +695,5 @@
 	    $inHeader->{FPA_ID}->{value} = $fpa_id;
 	}
-
+	
 	push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
 	push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq	(revision 29515)
@@ -24,5 +24,5 @@
 
 # list file columns
-# RA DEC FILTER MJD_MIN MJD_MAX
+# RA DEC FILTER MJD_MIN MJD_MAX | COMMENT
 
 my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
@@ -62,5 +62,5 @@
 GetOptions(
     'list=s'            => \$list,          # list of coordinates if undef ra and dec or x and y are required
-    'ra=s'              => \$ra,             # 
+    'ra=s'              => \$ra,            # 
     'dec=s'             => \$dec,
     'x=s'               => \$x,
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl	(revision 29515)
@@ -239,4 +239,10 @@
                     print "skipping $command\n";
                 }
+            } elsif ($chip->{fault}) {
+                # fault the dependent
+                my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
+            } elsif ($chip->{dsFile_fault}) {
+                # fault the dependent
+                my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
             }
         }
@@ -317,4 +323,5 @@
             if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
                 $chips_ready = 0;
+                $chip->{fault} = $chip->{chip_fault};
                 push @chipsToUpdate, $chip;
             } else {
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_finish.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_finish.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_finish.pl	(revision 29515)
@@ -191,5 +191,7 @@
             my $filter = $job_params->{filter};
             $filter = "0" if !$filter;
-            $exp_info = "0|0|0|$filter|0|0";
+            my $mjd_obs = $job_params->{mjd_obs};
+            $mjd_obs = "0" if !$mjd_obs;
+            $exp_info = "$mjd_obs|0|0|$filter|0|0";
         }
 
@@ -477,4 +479,12 @@
     my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
 
+    # dateobs is UTC convert to TAI
+    # XXX: Do this properly
+    if ($year >= 2009) {
+        $ticks += 34;
+    } else {
+        $ticks += 33;
+    }
+
     return 40587.0 + ($ticks/86400.);
 }
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_insert_request.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_insert_request.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_insert_request.pl	(revision 29515)
@@ -44,5 +44,5 @@
 
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
-my $pstampdump = can_run('pstampdump')  or (warn "Can't find pstampdump"  and $missing_tools = 1);
+my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -53,19 +53,25 @@
 my ($extname, $extver, $req_name);
 {
-    my $command = "$pstampdump -headeronly $tmp_req_file";
+    # get the header keywords of interest.
+    # Note that if it's a pstamp request then REQ_NAME should be defined.
+    # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
+    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
+if (0) {
+    # stoopid fields doesn't set exit status to zero when it works
     unless ($success) {
         print STDERR @$stderr_buf;
         exit $error_code >> 8;
     }
+}
     my $output = join "", @$stdout_buf;
-    ($extname, $extver, $req_name) = split " ", $output;
-    if ($extname and ($extname ne "PS1_PS_REQUEST")) {
+    (undef, $extname, $extver, $req_name) = split " ", $output;
+    if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
         print STDERR "invalid request file\n";
         exit $PS_EXIT_DATA_ERROR;
     }
     if (!defined $req_name) {
-        print STDERR "invalid request file no REQ_NAME\n";
+        print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
         exit $PS_EXIT_DATA_ERROR;
     }
@@ -91,6 +97,5 @@
 my $req_id = 0;
 {
-
-    my $command = "$pstamptool -addreq -name $req_name -uri $req_file -ds_id 0";
+    my $command = "$pstamptool -addreq -uri $req_file -ds_id 0 -name $req_name";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_job_run.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_job_run.pl	(revision 29515)
@@ -118,4 +118,5 @@
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
+    $command .= " -stage $params->{stage}" if $params->{stage};
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_server_status
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_server_status	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_server_status	(revision 29515)
@@ -16,4 +16,5 @@
 
 my ($rundir, $verbose, $save_temps);
+my $br = '<br />';
 
 GetOptions(
@@ -99,10 +100,10 @@
     print "Pantasks Scheduler $scheduler_state.\n";
     if ($controller_state) {
-        print "Pantasks Controller $controller_state.\n";
+        print $br . "Pantasks Controller $controller_state.\n";
     } else {
         print STDERR "Controller state not found";
         exit 1;
     }
-    print "\n";
+    print "$br$br\n";
 } else {
     print STDERR "Scheduler state not found";
@@ -116,5 +117,5 @@
 
 if ($request_run) {
-    print "<b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
+    print "<br /><b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
     if ($request_fin) {
         print "  $request_fin->{ngood} Requests finished.";
@@ -125,9 +126,18 @@
     print "\n";
     if ($job_run) {
-        print "<b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
-        print "\n";
+        print "<br /><b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
+        print "<br />\n";
     } else {
-        print "Task pstamp.job.run not found.\n";
+        print "Task pstamp.job.run not found.<br />\n";
     }
+    print "<br /><b>Unfinished Requests</b><br />\n";
+    print "<pre>\n";
+    system "/home/panstarrs/bills/ipp/tools/psstatus -r ";
+    print "</pre>\n";
+    print "<br /><b>Requests completed in last 24 hours (most recently completed first)</b><br />\n";
+    print "<pre>\n";
+    # finished requests
+    system "/home/panstarrs/bills/ipp/tools/psstatus -f";
+    print "</pre>\n";
 } else {
     print "Task pstamp.request.run not found.\n";
@@ -135,3 +145,5 @@
 
 
+
+
 exit 0;
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl	(revision 29515)
@@ -26,12 +26,8 @@
 my $dbname;
 my $dbserver;
-my $project;
-my $job_type;
 
 GetOptions(
-    'job_type=s'    =>  \$job_type,
     'dbname=s'      =>  \$dbname,
     'dbserver=s'    =>  \$dbserver,
-    'project=s'     => \$project,
     'verbose'       => \$verbose,
 );
@@ -42,11 +38,4 @@
     print "\n\n";
     print "Starting script $0 on $host\n\n";
-}
-
-my $listMode;
-if ($job_type and ($job_type eq 'list_uri')) {
-    $listMode=1;
-} else  {
-    $listMode=0;
 }
 
@@ -105,28 +94,4 @@
 }
 
-# ok at this point we have a request file add it to the database (unless we're in listMode)
-if ($listMode == 1 ) {
-    ###
-    ### In list mode just parse the file print the output and we're done
-    ###
-    my $command = "$pstampparse --mode list_uri --file $request_file";
-    $command .= " --dbname $dbname" if $dbname;
-    $command .= " --dbserver $dbserver" if $dbserver;
-    $command .= " --verbose" if $verbose;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if ($success) {
-        ### print "Matching Images:\n";
-        print @$stdout_buf;
-        exit 0;
-    } else {
-        # we send the output to STDOUT because that's where PHP finds it
-        print @$stdout_buf;
-        print @$stderr_buf;
-        exit 1;
-    }
-}
-
 # Queue the request
 my $req_id = 0;
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampparse.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampparse.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampparse.pl	(revision 29515)
@@ -805,9 +805,4 @@
 {
     my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
-
-    # XXX: The update process for warp and subsequent stages requires # destreaking to be performed
-    # because the -pending queries require the inputs to have magicked >= 0
-    # The case of stack-stack diffs not needing to be destreaked is taken care of in pstamp_checkdependent
-    $need_magic = 1 if $imagedb eq 'gpc1';
 
     # chipRun's can be in full state if destreaking is necessary
@@ -900,5 +895,5 @@
                     $$r_newState = 'stop';
                     $$r_fault = $PSTAMP_NOT_AVAILABLE;
-                } elsif (!$image->{magicked}) {
+                } elsif ($need_magic and !$image->{magicked}) {
                     $$r_newState = 'stop';
                     $$r_fault = $PSTAMP_NOT_DESTREAKED;
@@ -906,6 +901,7 @@
                     # cause the image to be re-made
                     # set up to queue an update run
+                    my $require_magic = ($need_magic or $image->{magicked});
                     get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
-                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
+                        $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
                 }
             }
Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampstopfaulted
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampstopfaulted	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstampstopfaulted	(revision 29515)
@@ -0,0 +1,15 @@
+
+# script executed by the task pstamp.stopfaulted to stop proceessing of jobs
+# and dependents that have faulted 5 or more times
+# all arguments to this script are passed to pstamptool
+
+# 25 is PSTAMP_NOT_AVAILABLE
+fault_code=25
+fault_count=3
+
+date
+echo stopping faulted dependent jobs
+pstamptool -stopdependentjob -set_fault $fault_code -fault_count $fault_count $*
+
+echo stopping faulted jobs
+pstamptool -updatejob -state run -set_state stop -set_fault $fault_code -fault_count $fault_count $*
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.c
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.c	(revision 29515)
@@ -22,5 +22,5 @@
 
     // define the active I/O files
-    if (!ppstampParseCamera(config)) {
+    if (!ppstampParseCamera(config, options)) {
         psErrorStackPrint(stderr, "Unable to parse camera.");
         ppstampCleanup(config, options);
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.h
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstamp.h	(revision 29515)
@@ -18,5 +18,5 @@
 
 // Determine what type of camera, and initialise
-bool ppstampParseCamera(pmConfig *config);
+bool ppstampParseCamera(pmConfig *config, ppstampOptions *options);
 
 int ppstampMakeStamp(pmConfig *config, ppstampOptions *);
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstampArguments.c
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstampArguments.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstampArguments.c	(revision 29515)
@@ -78,4 +78,9 @@
         psArgumentRemove(argnum, &argc, argv);
     }
+    if ((argnum = psArgumentGet(argc, argv, "-stage"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        options->stage = psStringCopy(argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    }
 
     pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist");
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstampMakeStamp.c
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstampMakeStamp.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstampMakeStamp.c	(revision 29515)
@@ -233,5 +233,10 @@
     int status = false;
 
-    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT");
+    pmFPAfile *output;
+    if (!options->stage || (strcmp(options->stage, "diff") != 0)) {
+        output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT");
+    } else {
+        output = psMetadataLookupPtr(NULL, config->files, "PPSTAMP.OUTPUT.DIFF");
+    }
     if (!output) {
         psError(PS_ERR_UNKNOWN, false, "Can't find output data\n");
@@ -674,4 +679,5 @@
 static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance)
 {
+#ifdef notdef
     bool status;
     psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
@@ -719,5 +725,14 @@
         }
     }
-    fprintf(stderr, "excised %ld masked pixels\n", numExcised);
+#endif
+
+    long numCensored;
+    if (!pmCensorMasked(config, image, mask, variance, &numCensored)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to censor masked pixels.\n");
+        return false;
+    }
+
+    // XXX: this shouldn't be a fprintf
+    fprintf(stderr, "Censored %ld masked pixels\n", numCensored);
 
     return true;
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstampOptions.h
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstampOptions.h	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstampOptions.h	(revision 29515)
@@ -10,4 +10,5 @@
     psString    chipName;
     psString    cellName;
+    psString    stage;
     //
     // Calculated Values
Index: /branches/eam_branches/ipp-20100823/pstamp/src/ppstampParseCamera.c
===================================================================
--- /branches/eam_branches/ipp-20100823/pstamp/src/ppstampParseCamera.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pstamp/src/ppstampParseCamera.c	(revision 29515)
@@ -8,7 +8,19 @@
 
 // Set up the ppstamp output Image file
-bool setupOutput(pmConfig *config, pmFPAfile *input, bool doMask, bool doWeight)
+bool setupOutput(pmConfig *config, pmFPAfile *input, psString stage, bool doMask, bool doWeight)
 {
-    pmFPAfile *output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT");
+    pmFPAfile *output;
+    
+    if (!stage || (strcmp(stage, "diff") != 0)) {
+        output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT");
+    } else {
+        // need special filerule for diff stage image to allow for negative values
+        output = pmFPAfileDefineSkycell(config, NULL, "PPSTAMP.OUTPUT.DIFF");
+    }
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to setup output.");
+        return false;
+    }
+
     output->save = true;
 
@@ -29,5 +41,5 @@
 // something else that I'm missing?
 
-bool ppstampParseCamera(pmConfig *config)
+bool ppstampParseCamera(pmConfig *config, ppstampOptions *options)
 {
     bool status = false;
@@ -83,5 +95,5 @@
 
     // Set up the output target
-    if (!setupOutput(config, input, doMask, doWeight)) {
+    if (!setupOutput(config, input, options->stage, doMask, doWeight)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to setup output.");
         return false;
Index: /branches/eam_branches/ipp-20100823/pswarp/src/pswarpTransformTile.c
===================================================================
--- /branches/eam_branches/ipp-20100823/pswarp/src/pswarpTransformTile.c	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/pswarp/src/pswarpTransformTile.c	(revision 29515)
@@ -78,5 +78,5 @@
     int yMax = PS_MIN(maxPt.y, outNumRows);
 
-    double jacobian = map->Xx * map->Yy - map->Yx * map->Xy; // Jacobian of transformation
+    double jacobian = fabs(map->Xx * map->Yy - map->Yx * map->Xy); // Jacobian of transformation
     double jacobian2 = PS_SQR(jacobian);                     // Square Jacobian
 
Index: /branches/eam_branches/ipp-20100823/tools/checkexp
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/checkexp	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/checkexp	(revision 29515)
@@ -20,5 +20,5 @@
 my $endDate;
 my $verbose;
-my $dbname = "gpc1";
+my $dbname;
 
 GetOptions(
@@ -33,4 +33,8 @@
 #           -exitval => 3)
 #    unless defined $startDate;
+
+if (!$dbname) {
+    my $dbname = "gpc1";
+}
 
 my $dbh  = getDBHandle($ipprc, $dbname);
Index: /branches/eam_branches/ipp-20100823/tools/czarclean.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czarclean.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czarclean.pl	(revision 29515)
@@ -9,14 +9,16 @@
 
 my $czarDbName = undef;
-my $begin = undef;
-my $end = undef;
+my $from = undef;
+my $to = undef;
 my $interval = undef;
 my $verbose = undef;
+my $optimize = undef;
 
 GetOptions (
         "dbname|d=s" => \$czarDbName,
         "interval|i=s" => \$interval,
-        "begin|b=s" => \$begin,
-        "end|e=s" => \$end,
+        "from|f=s" => \$from,
+        "to|t=s" => \$to,
+        "optimize|o" => \$optimize,
         "verbose|v" => \$verbose,
         );
@@ -30,19 +32,23 @@
     print "* UNKNKOWN: option                          @ARGV\n";
 }
-if (!$begin) {
+if (!$from) {
     $quit=1;
-    print "* REQUIRED: choose a begin date             -b <datetime>                (default=7am this morning)\n";
+    print "* REQUIRED: choose a from date                 -b <datetime>\n";
 }
-if (!$end) {
-    $end=$begin;
-    print "* OPTIONAL: choose an end date              -e <datetime>                (default=$end)\n";
+if (!$to) {
+    if($from) {$to=$from;} else {$to="NULL";}
+    print "* OPTIONAL: choose a to date                   -e <datetime>                   (default=$to)\n";
 }
 if (!$czarDbName) {
     $czarDbName = "czardb";
-    print "* OPTIONAL: choose czar Db name             -d <name>                    (default=$czarDbName\n";
+    print "* OPTIONAL: choose czar Db name                -d <name>                       (default=$czarDbName\n";
 }
 if (!$interval) {
     $interval = "30 MINUTE";
-    print "* OPTIONAL: choose time interval            -i <'1 hour'|'2 hour'|etc>   (default=$interval)\n";
+    print "* OPTIONAL: choose time interval               -i <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
+}
+if (!$optimize) {
+    $optimize = 0;
+    print "* OPTIONAL: optimize database after cleanup    -o <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
 }
 print "*\n*******************************************************************************\n";
@@ -59,4 +65,4 @@
 
 
-$czarDb->cleanupDateRange($begin, $end, $interval);
-
+$czarDb->cleanupDateRange($from, $to, $interval);
+if ($optimize) {$czarDb->optimize();}
Index: /branches/eam_branches/ipp-20100823/tools/czarplot.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czarplot.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czarplot.pl	(revision 29515)
@@ -17,8 +17,12 @@
 my $begin = undef;
 my $end = undef;
+my $day = undef;
 my $path = undef;
 my $verbose = undef;
 my $histogram = undef;
 my $timeSeries = undef;
+my $rate = undef;
+my $deriv = undef;
+my $showCleanup = undef;
 my $nebulous = undef;
 my $savingToFile = undef;
@@ -26,5 +30,5 @@
 
 GetOptions (
-        "dbname|d=s" => \$czarDbName,
+        "dbname=s" => \$czarDbName,
         "label|l=s" => \$label,
         "stage|s=s" => \$stage,
@@ -32,7 +36,11 @@
         "begin|b=s" => \$begin,
         "end|e=s" => \$end,
+        "day|y=s" => \$day,
         "output|o=s" => \$path,
         "histogram|h" => \$histogram,
         "nebulous|n" => \$nebulous,
+        "cleanup|c" => \$showCleanup,
+        "rate|r" => \$rate,
+        "deriv|d" => \$deriv,
         "timeseries|t" => \$timeSeries,
         "verbose|v" => \$verbose,
@@ -51,4 +59,12 @@
 if (!$timeSeries) {
     print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";} 
+if (!$rate) {
+    print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";} 
+if (!$deriv) {
+    $deriv = 0;
+    print "* OPTIONAL: plot first derivative           -d                          (default=$deriv)\n";} 
+if (!$showCleanup) {
+    $showCleanup = 0;
+    print "* OPTIONAL: include cleanup in timeseries   -c                          (default=$showCleanup)\n";} 
 if (!$log) {
     $log = 0;
@@ -64,7 +80,9 @@
     print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=none\n";} 
 if (!$begin) {
-    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=7am this morning)\n";} 
+    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=6:35am this morning)\n";} 
 if (!$end) {
     print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";} 
+if (!$day) {
+    print "* OPTIONAL: choose a single day to plot     -y <date>                   (default=today)\n";} 
 if (!$path) {
     print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
@@ -85,17 +103,36 @@
 
 
-# GENE PLOTS my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png size 1280,960 font \"/usr/share/fonts/corefonts/arial.ttf\" 12" : "X11", $path, $save_temps);
-my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps);
+my $plotter = new czartool::Plotter(
+        $czarDb, 
+        "%Y%m%d-%H%M%S", 
+        $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", 
+        $path, 
+        $save_temps);
 
 # sort out times
-if (!$end) {$end = $czarDb->getNowTimestamp();}
-if (!$begin) {
-    if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
-    else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
+if($day) {
+
+    # day plots should run from about 6:30am until midnight
+    $begin =  "$day 06:35";
+    $end = "$day 23:59";
 }
+else {
 
+    if (!$end) {$end = $czarDb->getNowTimestamp();}
+    if (!$begin) {
+
+        if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
+        else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
+    }
+}
 my $diskUsage = 1; # TODO
 
-if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log);}
+if ($rate) {
+
+    if (!$interval) {$interval = "1 HOUR";}
+    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $interval, $log);
+    exit;
+}
+if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);}
 if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
 if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
Index: /branches/eam_branches/ipp-20100823/tools/czarpoll.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czarpoll.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/czarpoll.pl	(revision 29515)
@@ -0,0 +1,280 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+use czartool::Nebulous;
+use czartool::Plotter;
+use czartool::Burntool;
+
+my $period = 60;
+my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
+my $save_temps = 0;
+
+GetOptions (
+        "period|p=s" => \$period, # TODO more Db args
+        "dbname|d=s" => \$czarDbName,
+        );
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $nebulous = new czartool::Nebulous($czarDb);
+my $pantasks = new czartool::Pantasks();
+my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
+my $burntool = new czartool::Burntool();
+
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+timePoll($period);
+
+###########################################################################
+#
+# Updates the labels from pantasks for all interested servers 
+#
+###########################################################################
+sub updateLabels {
+
+    print "* Updating labels\n";
+    my @servers = ("stdscience", "distribution", "publishing", "update");
+
+    my $server = undef;
+    foreach $server (@servers) {
+
+        my @labels = @{$pantasks->getLabels($server)};
+        if (@labels) { 
+        
+            $czarDb->updateCurrentLabels($server, \@labels);
+        }
+        else {
+        
+             print "WARNING: No labels to update for '$server'\n";
+        }
+    }
+}
+
+###########################################################################
+#
+# Updates pantasks server status TODO should really get info for all servers at once 
+#
+###########################################################################
+sub updateServerStatus {
+    print "* Checking all pantasks servers\n";
+
+    my $servers = $pantasks->getServerList();
+
+    my $server = undef;
+    my $alive = undef;
+    my $running = undef;
+    foreach $server (@{$servers}) {
+
+        $pantasks->getServerStatus($server, \$alive, \$running);
+        $czarDb->updateServerStatus($server, $alive, $running);
+    }
+}
+
+###########################################################################
+#
+# Polls with provided period (seconds) 
+#
+###########################################################################
+sub timePoll {
+    my ($period) = @_;
+
+    my $label;
+    my $new;
+    my $full;
+    my $faults;
+    my $stage;
+    my $query = undef;
+    my $str = undef;
+    my $labels = undef;
+    my $updateLabels = undef;
+    my $row = undef;
+    my $begin = undef;
+    my $end = undef;
+    my $priority = undef;
+    my $newState = undef;
+    my $nsStatus = undef;
+    my $lastDay = strftime('%Y-%m-%d', localtime);
+    my $today = undef;
+
+    while (1) {
+
+        # check whether day has changed. if so, cleanup tables from previous day and optimize
+        $today = strftime('%Y-%m-%d', localtime);
+        if ($czarDb->isBefore($lastDay, $today)) {
+        
+                print "* New day - performing cleanup\n";
+                $czarDb->cleanupDateRange($lastDay, $lastDay, "30 MINUTE");
+                $czarDb->optimize();
+                $lastDay = $today;
+        }
+
+        # sort out times
+        $begin = strftime('%Y-%m-%d 06:35',localtime);
+        $end = $czarDb->getNowTimestamp();
+
+        if ($czarDb->isBefore($end, $begin)) {
+
+            $begin = $czarDb->subtractInterval($begin, "1 DAY");
+        }
+
+        # check nightly science status
+        print "* Checking nightly science status\n";
+        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
+        $czarDb->updateNightlyScience($nsStatus);
+
+        # check nebulous
+        print "* Checking Nebulous\n";
+        $nebulous->updateClusterSpaceInfo();
+        $plotter->plotDiskUsageHistogram();
+        updateServerStatus();
+
+        # check labels
+        updateLabels();
+
+        # servers to check
+        my @serversToCheck = ("stdscience", "update");
+
+        my $thisServer = undef;
+        foreach $thisServer (@serversToCheck) {
+
+            if ($thisServer eq "update") {$newState = "update";}
+            else {$newState = "new";}
+
+            # deal with stdscience labels
+            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
+            my $size = @{$labels};
+            if($size > 0) {
+
+                # get priority
+                foreach $row ( @{$labels} ) {
+                    my ($label) = @{$row};
+                    $priority = $gpc1Db->getPriority($label);
+                    $czarDb->setLabelPriority($label, $priority);
+                }
+
+                updateAllStages($thisServer, $newState, $labels, $begin, $end);
+                createPlots($thisServer, $labels, $begin, $end);
+            }
+            else { print "* WARNING: no $thisServer labels found in Db\n";}
+        }
+
+        print "--------------------------------------------------------------------------\n";
+        print "* Going to sleep\n";
+        sleep($period);
+        print "* Waking up\n";
+    };
+}
+
+###########################################################################
+#
+# Loops through labels and creates time series and histogram plots
+#
+###########################################################################
+sub createPlots {
+    my ($server, $rows, $begin, $end) = @_;
+
+    my $stage = undef;
+    my $row = undef;
+
+    print "* Generating plots\n";
+
+    # create plots for each label for each stage
+    foreach $stage (@stages) {
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
+        }
+    }
+
+    # create plots for each label for all stages
+    foreach $row ( @{$rows} ) {
+        my ($label) = @{$row};
+
+        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
+        $plotter->createHistogram($label, $begin, $end);
+
+        #routineChecks($label, "1 HOUR");
+    }
+    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
+    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
+    foreach $stage (@stages) {
+
+        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
+    }
+}
+
+###########################################################################
+#
+# Loops through some labels and updates processed/pending/faults in the Db
+#
+###########################################################################
+sub updateAllStages {
+    my ($labelServer, $newState, $rows, $begin, $end) = @_;
+
+    print "* Updating stage data\n";
+    my $totalNew = undef;
+    my $totalFaults = undef;
+    my $totalFull = undef;
+    my $stage = undef;
+    my $reverting = 0;
+    my $row = undef;
+    my $new = undef;
+    my $full = undef;
+    my $faults = undef;
+    my $server = undef;
+    my $state = undef;
+    
+    foreach $stage (@stages) {
+
+        $server = $pantasks->getServerForThisStage($stage);
+        $pantasks->getRevertStatus($stage, \$reverting);
+        $czarDb->updateRevertStatus($stage, $reverting);
+
+        print "* Checking labels for $stage stage\n";
+
+        $totalNew=$totalFaults=$totalFull=0;
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            chomp($label);
+
+            if ($stage eq "burntool") {
+
+                if ($labelServer eq "stdscience") {
+
+                    $burntool->getPendingAndProcessed($label, \$new, \$full);
+                    $faults = 0;
+                }
+                else { $new = $full = $faults = 0;}
+            }
+            else {
+
+                $new = $gpc1Db->countExposures($label, $stage, $newState);
+                $full = $gpc1Db->countExposures($label, $stage, "full");
+                $faults = $gpc1Db->countFaults($label, $stage, $newState);
+            }
+            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
+            $totalNew += $new;
+            $totalFull += $full;
+            $totalFaults += $faults;
+
+            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
+        }
+
+        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
+    }
+}
+
Index: /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm	(revision 29515)
@@ -54,5 +54,5 @@
 
         chomp($line);
-        if($line =~ m/([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) {
+        if($line =~ m/BTSTATS: $today $target ([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) {
             if ($self->{_verbose}) {print "Output =  $1 $2 $3 $4\n";}
             ${$pending} = ($2 - $3)/60;
Index: /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm	(revision 29515)
@@ -11,10 +11,14 @@
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
 
-# Override constructor
+###########################################################################
+#
+# Constructor (overrides superclass)
+#
+###########################################################################
 sub new {
     my ($class) = @_;
 
-    # Call the constructor of the parent class, Person.
-    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4],  $_[5], $_[6]);
+    # Call the constructor of the parent class
+    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4], $_[5], $_[6]);
 
     bless $self, $class;
@@ -39,9 +43,5 @@
 SQL
 
-
-    if (!$query->execute) {
-
-        return 0;
-    }
+    if (!$query->execute) {return 0;}
 
     ${$labels} = $query->fetchall_arrayref();
@@ -238,18 +238,12 @@
 ###########################################################################
 #
-# Gets time series data and stores it to temp file
-#
-###########################################################################
-sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_;
-
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
-
-    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+# Gets time min/max/diff for a given period
+#
+###########################################################################
+sub getTimeMinMaxDiff {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        MIN(processed), GREATEST(MAX(pending), MAX(faults), max(processed)-min(processed)) AS maxY, 
-        LEAST(MIN(pending), MIN(faults), max(processed)-min(processed)) AS minY,
         DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
         DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
@@ -262,17 +256,38 @@
     if (!$query->execute) {return 0;}
 
-    my $minProcessed = undef;
-    my ($_maxY, $_minY, $_timeDiff);
-    ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
-
-    if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;}
-    if ($_minY < ${$minY}) {${$minY} = $_minY;}
+    my ($_timeDiff);
+    (${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
+
     if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
 
+return 1;
+}
+
+
+###########################################################################
+#
+# Gets time series data and stores it to temp file
+#
+###########################################################################
+sub createTimeSeriesData {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MIN(processed)
+        FROM $stage 
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    if (!$query->execute) {return 0;}
+    my $minProcessed = scalar  $query->fetchrow_array(); # TODO remove
     if (!defined $minProcessed) {return 0;}
+
+    $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
 
     $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed-$minProcessed 
+        timestamp, pending, faults, processed 
         FROM $stage 
         WHERE label LIKE '$label' 
@@ -283,16 +298,74 @@
     $query->execute;
 
-    # loop round results
+    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    my $lastProcessed = -1;
+    my $lastPending = -1;
+    my $lastFaults = -1;
+    my $lastTimestamp = undef;
+    my $processed = 0;
+    my $runningProcessed = 0;
+    my $pending = 0;
+    my $faults = 0;
+    my $timestamp = undef;
     while (my @row = $query->fetchrow_array()) {
-        my ($timestamp, $pending, $faults, $processed) = @row;
-
-        if ($isLog) {
-
-            $processed = log($processed + 1);
-            $pending = log($pending + 1);
-            $faults = log($faults + 1);
-        }
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        if ($showCleanup ) {
+        
+            $runningProcessed = $thisProcessed - $minProcessed;
+        }
+        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $runningProcessed = $runningProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+
+        if ($firstDeriv) {
+
+            if (defined $lastTimestamp) {
+
+                my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+                
+                if ($thisProcessed > $lastProcessed){
+                $processed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+#print "$processed = ($thisProcessed - $lastProcessed)/$timeDiff\n";
+                }
+                else {$processed = 0;}
+                #$pending = abs($thisPending - $lastPending)/$timeDiff;    
+                #$faults = abs($thisFaults - $lastFaults)/$timeDiff;    
+            }
+            else {
+                $processed = 0;
+                $pending = 0;
+                $faults = 0; 
+            }
+        }
+        elsif ($isLog) {
+
+            $processed = ($runningProcessed < 0) ? 0 : $runningProcessed;
+            $pending =  ($thisPending < 0) ? 0 : $thisPending;
+            $faults =  ($thisFaults < 0) ? 0 : $thisFaults;
+
+            $processed = log($runningProcessed + 1)/log(10);
+            $pending = log($thisPending + 1)/log(10);
+            $faults = log($faults + 1)/log(10);
+        }
+        else {
+        
+            $processed = $runningProcessed;
+            $pending = $thisPending;
+            $faults = $thisFaults;
+        }
+
+        $timestamp = $self->getFormattedDate($thisTimestamp);
 
         print GNUDAT "$timestamp $pending $faults $processed\n";
+
+        $lastProcessed = $thisProcessed;
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $lastTimestamp = $thisTimestamp;
     }
 
@@ -357,5 +430,4 @@
 sub countProcessedPendingAndFaults {
     my ($self, $label, $stage, $fromTime, $toTime, $processed, $pending, $faults) = @_;
-
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -371,13 +443,7 @@
     (${$pending}, ${$faults}) = $query->fetchrow_array();    
 
-    $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        MAX(processed) - MIN(processed) 
-        FROM $stage 
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
-SQL
-    $query->execute;
-    (${$processed}) = $query->fetchrow_array();
+    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
+
+    return 1;
 }
 
@@ -502,29 +568,239 @@
         print GNUDAT "$timestamp $available\n";
     }
+
     close(GNUDAT);
 
     return $dataFile;
 }
-###########################################################################
-#
-# Determines how much has been processed in the provided interval of time 
-# (format: 1 HOUR, 1 MINUTE 1 DAY etc)
-#
-###########################################################################
-sub countProcessed { # TODO use time not interval
-    my ($self, $label, $stage, $interval) = @_;
+
+###########################################################################
+#
+# TODO implement isLog
+#
+###########################################################################
+sub createProcessingRateData {
+    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
+
+    my $startTime = $startDay;
+    my $endTime;
+    my $quit = 0;
+    my $processed;
+    my $pending;
+    my $faults;
+    my $timestamp;
+    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+    my $cleanupCarry = 0;
+    while(1) {
+
+        if (!$self->isBefore($startTime, $endDay)) {last;}
+        $endTime = $self->addInterval($startTime, $interval);
+        $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
+        $timestamp = $self->getFormattedDate($endTime);
+        print GNUDAT "$timestamp $processed 0 0\n";
+
+        $startTime = $endTime;
+    }
+
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
+}
+
+###########################################################################
+#
+# When did this stage finish? 
+#
+###########################################################################
+sub getFinishTime {
+    my ($self, $label, $stage, $begin, $end) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        MAX(processed) - MIN(processed) 
-        FROM $stage 
+        timestamp, pending, faults
+        FROM $stage
         WHERE label LIKE '$label' 
-        AND timestamp > (now() - INTERVAL $interval); 
-SQL
-        $query->execute;
-
-    return scalar $query->fetchrow_array();
-}
-
+        AND timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+SQL
+    $query->execute;
+
+    my $array = $query->fetchall_arrayref();
+
+    my $row;
+    my $thisPending = -1;
+    my $lastPending = -1;
+    my $thisFaults = -1;
+    my $lastFaults = -1;
+    my $timestamp;
+    my $started = undef;
+    my $finished = undef;
+    my $thisLeft = 0;
+    my $lastLeft = 0;
+    my $finishTimeout = "01:00:00";
+    my $numRows = @{$array};
+    print "$numRows rows\n";
+    my $n = 0;
+    foreach $row ( @{$array} ){
+
+        ($timestamp, $thisPending, $thisFaults) = @{$row};
+
+#        print "$timestamp, $thisPending, $thisFaults\n";
+
+        $thisLeft = $thisPending - $thisFaults;
+        $lastLeft = $lastPending - $lastFaults;
+
+
+        if ($n == 0 && $thisLeft > 0) {
+
+            print "Starting this time period with stuff pending ($thisLeft)\n";
+            $started = $timestamp;
+        }
+
+        if (!defined $started && $lastLeft == 0 && $thisLeft > 0) {
+
+            $started = $timestamp;
+            print "STARTED at $started\n";
+        }
+
+        if ($started && !defined $finished && $thisLeft == 0) {
+
+            $finished = $timestamp;
+            print "setting FINISHED to $finished\n";
+        }
+
+        if (defined $finished) {
+
+            if ($thisLeft != 0) {
+                
+                $finished = undef;
+            
+                print "NEW pending  at $timestamp\n";
+            
+            }
+            elsif ($thisLeft == 0) {
+                
+                my $howLongFinished = $self->diffTimes($timestamp, $finished); 
+
+print "$howLongFinished = $timestamp, $finished\n";
+
+                if ($self->isIntervalGreaterThan($howLongFinished, $finishTimeout)) {
+                    
+                    print "0 pending for $howLongFinished. We're done \n";
+                    last;
+                }
+
+            }
+
+        }
+
+
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $n++;
+        if ($n == $numRows) {
+
+            if ($thisLeft == 0) {
+
+                print "exceeded interval before reaching finished timeout\n";
+            }
+            else {
+                print "Reached end of interval and not finished yet. $thisLeft left\n";
+            }
+        }
+    }
+
+    if ($finished) {
+
+        my $timeTaken = $self->diffTimes($finished, $started);
+        print "FINISHED at $finished with $thisFaults remaining faults, time taken = $timeTaken\n";
+    }
+
+}
+
+###########################################################################
+#
+# Figures out if a stage has plateaued 
+#
+###########################################################################
+sub hasPlateaued {
+    my ($self, $label, $stage, $end, $interval) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        timestamp, pending, faults
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$end - INTERVAL $interval'
+        AND timestamp <= '$end'
+        ORDER BY timestamp DESC;
+SQL
+    $query->execute;
+
+    my $array = $query->fetchall_arrayref();
+
+    my $thisPending = -1;
+    my $lastPending = -1;
+    my $thisFaults = -1;
+    my $lastFaults = -1;
+    my $timestamp;
+    my $thisLeft = 0;
+    my $lastLeft = 0;
+    my $row;
+    my $numRows = @{$array};
+    my $n = 0;
+    foreach $row ( @{$array} ){
+
+        ($timestamp, $thisPending, $thisFaults) = @{$row};
+
+        $thisLeft = $thisPending - $thisFaults;
+        $lastLeft = $lastPending - $lastFaults;
+
+        print "$timestamp $thisPending\n";
+
+        if ($n>0) {
+
+            #if () {}
+
+        }
+
+        $lastPending = $thisPending;
+        $lastFaults = $thisFaults;
+        $n++;
+    }
+}
+
+###########################################################################
+#
+# Gets count of processed stuff in a given time period 
+#
+###########################################################################
+sub countProcessed {
+    my ($self, $label, $stage, $begin, $end) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        processed 
+        FROM $stage
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$begin' 
+        AND timestamp < '$end' 
+SQL
+        $query->execute;
+
+    my $processedArray = $query->fetchall_arrayref();
+
+    my $processed;
+    my $thisCount = -1;
+    my $lastCount = -1;
+    my $count = 0;
+    foreach $processed ( @{$processedArray} ){
+
+        ($thisCount) = @{$processed};
+
+        if ($thisCount > $lastCount && $lastCount != -1) {$count = $count + ($thisCount -  $lastCount);}
+        $lastCount = $thisCount;
+    }
+
+    return $count;
+}
 
 ###########################################################################
@@ -541,5 +817,5 @@
 
         if (!$self->isBefore($thisDay, $endDay)) {
-        
+
             $quit = 1;
         }
@@ -553,5 +829,6 @@
 ###########################################################################
 #
-# Deletes all but one row per interval from all stage tables for all labels between the provided day
+# Deletes all but one row per interval from all tables for the provided date range
+# TODO this is very clumsy, I just have time to thinmk of something more elegant
 #
 ###########################################################################
@@ -566,13 +843,14 @@
     my $totalDeleted = undef;
     my $quit = 0;
+    my $query = undef;
     while(!$quit) {
 
         $toTime = $self->addInterval($fromTime, $interval);
         if (!$self->isBefore($toTime, $endDay)) {
-        
+
             $toTime = $endDay;
             $quit = 1;
         }
-   
+
         my $stage = undef;
         $totalDeleted = 0;
@@ -586,5 +864,5 @@
                 my ($label) = @{$row};
 
-                my $query = $self->{_db}->prepare(<<SQL);
+                $query = $self->{_db}->prepare(<<SQL);
                 SELECT COUNT(*) 
                     FROM $stage 
@@ -609,8 +887,144 @@
                 $totalDeleted += $toDelete;
             }
-        }
+
+        }
+
+        # servers table
+        my $servers = undef;
+        if ($self->getServers(\$servers)) {
+
+            my $server = undef;
+            my $row = undef;
+            foreach $row ( @{$servers} ) {
+                my ($server) = @{$row};
+
+                $query = $self->{_db}->prepare(<<SQL);
+                SELECT COUNT(*) 
+                    FROM servers 
+                    WHERE timestamp > '$fromTime'
+                    AND timestamp <= '$toTime'
+                    AND server = '$server' 
+SQL
+                    $query->execute;
+
+                my $toDelete = scalar $query->fetchrow_array() - 1;
+                if ($toDelete > 0) {
+
+                    $query = $self->{_db}->prepare(<<SQL);
+                    DELETE FROM servers 
+                        WHERE timestamp > '$fromTime' 
+                        AND timestamp <= '$toTime' 
+                        AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                        $query->execute;
+
+                    $totalDeleted += $toDelete;
+                }
+            }
+        }
+
+        # now deal with cluster_space table
+        $query = $self->{_db}->prepare(<<SQL);
+        SELECT COUNT(*) 
+            FROM cluster_space 
+            WHERE timestamp > '$fromTime'
+            AND timestamp <= '$toTime'
+SQL
+            $query->execute;
+
+        my $toDelete = scalar $query->fetchrow_array() - 1;
+        if ($toDelete > 0) {
+
+            $query = $self->{_db}->prepare(<<SQL);
+            DELETE FROM cluster_space 
+                WHERE timestamp > '$fromTime' 
+                AND timestamp <= '$toTime' 
+                ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                $query->execute;
+
+            $totalDeleted += $toDelete;
+        }
+
         print "   * Deleted $totalDeleted between $fromTime and  $toTime\n";
         $fromTime = $toTime;
     }
+}
+
+###########################################################################
+#
+# Optimizes all tables that need to be optimized 
+#
+###########################################################################
+sub optimize {
+    my ($self) = @_;
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+        $self->optimizeTable($stage);
+    }
+
+    $self->optimizeTable("cluster_space");
+    $self->optimizeTable("servers");
+}
+
+###########################################################################
+#
+# Returns if a particular server has been down for a certain interval 
+#
+###########################################################################
+sub isServerDown {
+    my ($self, $server, $interval) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM servers
+        WHERE server = "$server" 
+        AND timestamp > now() - INTERVAL $interval
+SQL
+
+        $query->execute;
+    my $numOfReadings = scalar $query->fetchrow_array();
+
+    if ($numOfReadings == 0) {return 0;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM servers
+        WHERE server = "$server" 
+        AND timestamp > now() - INTERVAL $interval
+        AND !running
+SQL
+
+        $query->execute;
+    my $numNotRunning = scalar $query->fetchrow_array();
+
+    if ($numOfReadings == $numNotRunning) {return 1;}
+
+    return 0;
+}
+
+###########################################################################
+#
+# Returns an array of servers 
+#
+###########################################################################
+sub getServers {
+    my ($self, $servers) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT server
+        FROM servers
+SQL
+
+        if (!$query->execute) {
+
+            return 0;
+        }
+
+    ${$servers} = $query->fetchall_arrayref();
+
+    return 1;
 }
 
@@ -682,5 +1096,5 @@
     INSERT INTO revision (revision) VALUES ($revision);
 SQL
-    $query->execute;
+        $query->execute;
 }
 
@@ -701,5 +1115,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
     my @row = $query->fetchrow_array();
 
@@ -729,5 +1143,5 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         CREATE TABLE $stage (
                 timestamp TIMESTAMP DEFAULT NOW(),
@@ -758,5 +1172,5 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         ALTER TABLE $stage 
             ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
@@ -783,10 +1197,10 @@
     foreach $stage (@stages) {
 
-    my $query = $self->{_db}->prepare(<<SQL);
+        my $query = $self->{_db}->prepare(<<SQL);
         ALTER TABLE $stage 
             ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
 SQL
 
-      $query->execute;
+            $query->execute;
     }
 
@@ -808,5 +1222,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
 
     $self->setRevision(4);
@@ -830,5 +1244,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
 
     $self->setRevision(5);
@@ -854,12 +1268,12 @@
 SQL
 
-      $query->execute;
-    }
-
-        my $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX serverIndex ON servers (timestamp, server);
-SQL
-
-      $query->execute;
+            $query->execute;
+    }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX serverIndex ON servers (timestamp, server);
+SQL
+
+        $query->execute;
 
 
@@ -882,5 +1296,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     $self->setRevision(7);
@@ -906,5 +1320,5 @@
 SQL
 
-    $query->execute;
+            $query->execute;
     }
 
@@ -916,5 +1330,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     # insert stages into revert table
@@ -926,5 +1340,5 @@
             ('$stage', 0);
 SQL
-       $query->execute;
+            $query->execute;
     }
 
@@ -951,5 +1365,5 @@
 SQL
 
-      $query->execute;
+        $query->execute;
     $query = $self->{_db}->prepare(<<SQL);
     CREATE TABLE hosts (
@@ -960,11 +1374,11 @@
 SQL
 
-      $query->execute;
-
-        $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
-SQL
-
-      $query->execute;
+        $query->execute;
+
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
+SQL
+
+        $query->execute;
 
     $self->setRevision(9);
@@ -984,8 +1398,8 @@
     ALTER TABLE hosts 
         ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
-        ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
-SQL
-
-    $query->execute;
+            ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
+SQL
+
+        $query->execute;
 
     $self->setRevision(10);
@@ -1012,10 +1426,10 @@
 SQL
 
-      $query->execute;
-        $query = $self->{_db}->prepare(<<SQL);
-        CREATE INDEX burntoolIndex ON burntool (timestamp, label);
-SQL
-
-      $query->execute;
+        $query->execute;
+    $query = $self->{_db}->prepare(<<SQL);
+    CREATE INDEX burntoolIndex ON burntool (timestamp, label);
+SQL
+
+        $query->execute;
 
     $self->setRevision(11);
@@ -1039,5 +1453,5 @@
 SQL
 
-    $query->execute;
+        $query->execute;
 
     $self->setRevision(12);
Index: /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm	(revision 29515)
@@ -64,4 +64,19 @@
     return $self->{_dbHost};                                                 
 }                                                                               
+###########################################################################
+#
+# Adds the provided interval to the provided time
+#
+###########################################################################
+sub getFormattedDate {
+    my ($self, $time) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DATE_FORMAT('$time','$self->{_dateFormat}'); 
+SQL
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
 
 ###########################################################################
@@ -83,4 +98,52 @@
 ###########################################################################
 #
+# Finds the difference of two times
+#
+###########################################################################
+sub diffTimes {
+    my ($self, $time1, $time2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIMEDIFF('$time1','$time2'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Finds the difference of two times in seconds
+#
+###########################################################################
+sub diffTimesInSecs {
+    my ($self, $time1, $time2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2')); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns whether the first interval is larger than the second
+#
+###########################################################################
+sub isIntervalGreaterThan {
+    my ($self, $interval1, $interval2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT INTERVAL('$interval1', '$interval2'); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
 # Subtracts the provided interval from the provided time
 #
@@ -127,4 +190,25 @@
 
 return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Optimizes a table 
+#
+#######################################################################################
+sub optimizeTable {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    OPTIMIZE TABLE $table; 
+SQL
+
+    my $success = $query->execute;
+
+    print "* ";
+    if (!$success) {print "UN";}
+    print "successfully optimized '$table' table\n";
+
+    return $success;
 }
 
Index: /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm	(revision 29515)
@@ -33,5 +33,5 @@
 ###########################################################################
 sub createImageFileName {
-    my ($self, $label, $stage, $suffix, $isLog) = @_;
+    my ($self, $label, $stage, $suffix, $isLog, $isRate) = @_;
 
     my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
@@ -55,16 +55,84 @@
 ###########################################################################
 #
+# Plots a time series of processing rate for all stages for this label TODO duplication below. combine
+#
+###########################################################################
+sub createRateTimeSeries {
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
+
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = -1;
+
+    my $stages = undef;                 
+
+    if (!$selectedStage) {$stages = \@allStages;}
+    else {$stages = ["$selectedStage"]};        
+
+    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
+
+    my $divX;
+    my $timeFormat;
+    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
+
+    my %gnuplotFiles;
+    my $stage = undef;
+    my $gnuplotFile = undef;
+    my $outputFile = createImageFileName($self, $label, $selectedStage, "r", $isLog);
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
+        #"set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+       "set format x \"$timeFormat\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+#        "set xrange [\"$minX\":\"$maxX\"];" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures processed per $interval\";" .
+        "set boxwidth 0.75;" .
+        "plot ";
+
+    my $first = 1;
+    foreach $stage (@{$stages}) {
+
+        if ($self->{_czarDb}->createProcessingRateData(
+                    $stage, 
+                    $label, 
+                    $beginTime, 
+                    $endTime, 
+                    $interval,
+                    \$gnuplotFile,
+                    $isLog)) {
+
+            $gnuplotFiles{$stage} = $gnuplotFile;
+
+            if (!$first) { print GP ","; }
+            print GP "'$gnuplotFile' using 2:xtic(1) title \"$stage\" ";
+            $first = 0;
+        }
+    }
+
+    print GP ";\n";
+    close GP;
+}                                                    
+###########################################################################
+#
 # Plots a time series for all stages for this label
 #
 ###########################################################################
 sub createTimeSeries {
-    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog) = @_;
-
-    my ($minX, $maxX, $minY, $maxY, $timeDiff);
-    $minX = 999999999;      
-    $maxX = -9999999999;
-    $minY = 999999999;          
-    $maxY = -99999999;
-    $timeDiff = 0;
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_;
+
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = 0;
 
     my $stages = undef;                 
@@ -85,9 +153,9 @@
                     \$minX, 
                     \$maxX, 
-                    \$minY, 
-                    \$maxY, 
                     \$timeDiff, 
                     \$gnuplotFile,
-                    $isLog)) {
+                    $isLog,
+                    $showCleanup,
+                    $deriv)) {
 
             $gnuplotFiles{$stage} = $gnuplotFile;
@@ -118,8 +186,7 @@
             $maxX, 
             $minX, 
-            $maxY, 
-            $minY, 
             $timeDiff, 
-            $isLog);
+            $isLog,
+            $deriv);
 }                                                    
 
@@ -161,8 +228,29 @@
     close(GNUDAT);
 
-    plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime, $maxY);
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($maxY == 0) {$maxY = 1;}
+    else {$maxY = $maxY*1.1;}
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"'$label', '$beginTime' to '$endTime'\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set yrange [\"0\":\"$maxY\"];" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures\";" .
+        "set boxwidth 0.75;" .
+        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
+        "\n";
+
+    close GP;
     unlink($inputFile);
 }
-
 
 ###########################################################################
@@ -239,20 +327,22 @@
 ###########################################################################
 sub plotTimeSeries {
-    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, $isLog) = @_;
+    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog, $isDeriv) = @_;
 
     my $timeFormat = undef;
     my $divX = undef;
     my $yTitle = undef;
-    if ($isLog) {$yTitle = "Log( numExposures )";}
-    else {$yTitle = "numExposures";}
+    if ($isLog) {$yTitle = "Log( Exposures )";}
+    elsif ($isDeriv) {$yTitle = "dExposures/dTime";}
+    else {$yTitle = "Exposures";}
 
     $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
 
     my $numOfPlots = keys %$gnuplotFiles;
-    my $title = undef;
 
     # sort out plot title
-    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}}
-    else {$title = "'All stages'"}
+    my $title = "";
+    if ($isDeriv) {$title .= "First derivatives of "}
+    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
+    else {$title .= "'all stages'"}
 
     $title .= " for '$label', '$fromTime' to '$toTime'";
@@ -283,6 +373,6 @@
         if ($numOfPlots == 1) {
 
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
-            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7 lw 2";
         }
@@ -346,37 +436,4 @@
     close GP;
     unlink($gnuplotFile);
-}
-
-###########################################################################
-#
-# Plots a histogram of processed stuff
-#
-###########################################################################
-sub plotHistogram {
-    my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime, $maxY) = @_;
-
-    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
-    use FileHandle;
-    GP->autoflush(1);
-
-    if ($maxY == 0) {$maxY = 1;}
-    else {$maxY = $maxY*1.1;}
-
-    print GP
-        "set term $self->{_outputFormat};" .
-        "set output \"$outputFile\";" .
-        "set title \"'$label', '$fromTime' to '$toTime'\";" .
-        "set grid;" .
-        "set boxwidth;" .
-        "set yrange [\"0\":\"$maxY\"];" .
-        "set style data histogram;" .
-        "set style histogram rowstacked;" .
-        "set style fill solid border -1;" .
-        "set ylabel \"Exposures\";" .
-        "set boxwidth 0.75;" .
-        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
-        "\n";
-
-    close GP;
 }
 
Index: /branches/eam_branches/ipp-20100823/tools/definetargets
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/definetargets	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/definetargets	(revision 29515)
@@ -33,5 +33,5 @@
 
 my @all_filters = qw( g r i z y ); 
-my @all_stages = qw( raw chip camera fake warp stack diff );
+my @all_stages = qw( raw chip camera fake warp stack diff SSdiff );
 
 my @filters;
Index: /branches/eam_branches/ipp-20100823/tools/makedistdest
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/makedistdest	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/makedistdest	(revision 29515)
@@ -0,0 +1,37 @@
+#!/usr/bin/env perl
+
+# program to Create a new distribution data store product and add it to the database
+# Note: The list of names of the product specific columns is set here.
+# The contents are set in dist_make_fileset.pl. 
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+my $prod_name;
+
+GetOptions(
+           'prod_name=s'     => \$prod_name,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --prod_name",
+           -exitval => 3)
+    unless defined $prod_name;  
+
+my $cmd = "dsprodtool --add $prod_name --type ipp-dist --description $prod_name";
+$cmd .= " --ps0 target_id --ps1 stage --ps2 stage_id --ps3 fs_tag --ps4 data_group --ps5 filter";
+
+print "$cmd\n";
+
+my $rc = system $cmd;
+die "dsprodtool failed: $rc" if $rc;
+
+$cmd = "disttool -dbname gpc1 -definedestination -ds_dbname ippRequestServer -ds_dbhost ippdb02 -name $prod_name";
+
+$rc = system $cmd;
+die "disttool failed: $rc" if $rc;
+
Index: /branches/eam_branches/ipp-20100823/tools/roboczar.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/roboczar.pl	(revision 29514)
+++ /branches/eam_branches/ipp-20100823/tools/roboczar.pl	(revision 29515)
@@ -14,296 +14,50 @@
 use czartool::Burntool;
 
-my $period = 60;
 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
 my $save_temps = 0;
 
 GetOptions (
-        "period|p=s" => \$period, # TODO more Db args
         "dbname|d=s" => \$czarDbName,
         );
 
 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
-my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
-my $nebulous = new czartool::Nebulous($czarDb);
-my $pantasks = new czartool::Pantasks();
-my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
-my $burntool = new czartool::Burntool();
 
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
 
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration");
 
 
-timePoll($period);
+while(1) {
 
-###########################################################################
-#
-# Updates the labels from pantasks for all interested servers 
-#
-###########################################################################
-sub updateLabels {
-
-    print "* Updating labels\n";
-    my @servers = ("stdscience", "distribution", "publishing", "update");
-
-    my $server = undef;
-    foreach $server (@servers) {
-
-        my @labels = @{$pantasks->getLabels($server)};
-        if (@labels) { 
-        
-            $czarDb->updateCurrentLabels($server, \@labels);
-        }
-        else {
-        
-             print "WARNING: No labels to update for '$server'\n";
-        }
-    }
+    checkServers("20 MINUTE");
+    sleep(1200);
 }
 
 ###########################################################################
 #
-# Updates pantasks server status TODO should really get info for all servers at once 
+# Checks tha the important servers are running 
 #
 ###########################################################################
-sub updateServerStatus {
-    print "* Checking all pantasks servers\n";
+sub checkServers {
+    my ($interval) = @_;
 
-    my $servers = $pantasks->getServerList();
+    my $server;
+    foreach $server (@serversWeCareAbout) {
 
-    my $server = undef;
-    my $alive = undef;
-    my $running = undef;
-    foreach $server (@{$servers}) {
+        if ($czarDb->isServerDown($server, $interval)) {
 
-        $pantasks->getServerStatus($server, \$alive, \$running);
-        $czarDb->updateServerStatus($server, $alive, $running);
-    }
-}
-
-###########################################################################
-#
-# Polls with provided period (seconds) 
-#
-###########################################################################
-sub timePoll {
-    my ($period) = @_;
-
-    my $label;
-    my $new;
-    my $full;
-    my $faults;
-    my $stage;
-    my $query = undef;
-    my $str = undef;
-    my $labels = undef;
-    my $updateLabels = undef;
-    my $row = undef;
-    my $begin = undef;
-    my $end = undef;
-    my $priority = undef;
-    my $newState = undef;
-    my $nsStatus = undef;
-
-    while (1) {
-
-        # sort out times
-        $begin =  strftime('%Y-%m-%d 06:35',localtime);
-        $end = $czarDb->getNowTimestamp();
-
-        if ($czarDb->isBefore($end, $begin)) {
-
-            $begin = $czarDb->subtractInterval($begin, "1 DAY");
+            print "$server has been down for the last $interval\n";
+            sendEmail(
+                    "roydhenderson\@gmail.com", 
+                    "roboczar\@ipp.com", 
+                    "Roboczar update", 
+                    "\n\n* '$server' server has been down for the last $interval\n\n");
         }
-
-        # check nightly science status
-        print "* Checking nightly science status\n";
-        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
-        $czarDb->updateNightlyScience($nsStatus);
-
-        # check nebulous
-        print "* Checking Nebulous\n";
-        $nebulous->updateClusterSpaceInfo();
-        $plotter->plotDiskUsageHistogram();
-        updateServerStatus();
-
-        # check labels
-        updateLabels();
-
-        # servers to check
-        my @serversToCheck = ("stdscience", "update");
-
-        my $thisServer = undef;
-        foreach $thisServer (@serversToCheck) {
-
-            if ($thisServer eq "update") {$newState = "update";}
-            else {$newState = "new";}
-
-            # deal with stdscience labels
-            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
-            my $size = @{$labels};
-            if($size > 0) {
-
-                # get priority
-                foreach $row ( @{$labels} ) {
-                    my ($label) = @{$row};
-                    $priority = $gpc1Db->getPriority($label);
-                    $czarDb->setLabelPriority($label, $priority);
-                }
-
-                updateAllStages($thisServer, $newState, $labels, $begin, $end);
-                createPlots($thisServer, $labels, $begin, $end);
-            }
-            else { print "* WARNING: no $thisServer labels found in Db\n";}
-        }
-
-        print "--------------------------------------------------------------------------\n";
-        print "* Going to sleep\n";
-        sleep($period);
-        print "* Waking up\n";
-
-        #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
-    };
-}
-
-###########################################################################
-#
-# Loops through labels and creates time series and histogram plots
-#
-###########################################################################
-sub createPlots {
-    my ($server, $rows, $begin, $end) = @_;
-
-    my $stage = undef;
-    my $row = undef;
-
-    print "* Generating plots\n";
-
-    # create plots for each label for each stage
-    foreach $stage (@stages) {
-        foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
-
-            chomp($label);
-            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
+        else {
+            #print "$server has been running for some of the last $interval\n";
         }
     }
-
-    # create plots for each label for all stages
-    foreach $row ( @{$rows} ) {
-        my ($label) = @{$row};
-
-        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
-        $plotter->createHistogram($label, $begin, $end);
-
-        #routineChecks($label, "1 HOUR");
-    }
-    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
-    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
-    foreach $stage (@stages) {
-
-        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
-    }
-}
-
-###########################################################################
-#
-# Loops through some labels and updates processed/pending/faults in the Db
-#
-###########################################################################
-sub updateAllStages {
-    my ($labelServer, $newState, $rows, $begin, $end) = @_;
-
-    print "* Updating stage data\n";
-    my $totalNew = undef;
-    my $totalFaults = undef;
-    my $totalFull = undef;
-    my $stage = undef;
-    my $reverting = 0;
-    my $row = undef;
-    my $new = undef;
-    my $full = undef;
-    my $faults = undef;
-    my $server = undef;
-    my $state = undef;
-    
-    foreach $stage (@stages) {
-
-        $server = $pantasks->getServerForThisStage($stage);
-        $pantasks->getRevertStatus($stage, \$reverting);
-        $czarDb->updateRevertStatus($stage, $reverting);
-
-        print "* Checking labels for $stage stage\n";
-
-        $totalNew=$totalFaults=$totalFull=0;
-        foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
-
-            chomp($label);
-
-            if ($stage eq "burntool") {
-
-                if ($labelServer eq "stdscience") {
-
-                    $burntool->getPendingAndProcessed($label, \$new, \$full);
-                    $faults = 0;
-                }
-                else { $new = $full = $faults = 0;}
-            }
-            else {
-
-                $new = $gpc1Db->countExposures($label, $stage, $newState);
-                $full = $gpc1Db->countExposures($label, $stage, "full");
-                $faults = $gpc1Db->countFaults($label, $stage, $newState);
-            }
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
-            $totalNew += $new;
-            $totalFull += $full;
-            $totalFaults += $faults;
-
-            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
-        }
-
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
-    }
-}
-
-###########################################################################
-#
-# Performs some routine checks on processing status and sends alerts if it needs to 
-#
-###########################################################################
-sub routineChecks {
-    my ($label, $interval) = @_;
-
-    my $faultsNow;
-    my $faultsInPast;
-    my $newFaults;
-    my $pendingNow;
-    my $processedRecently;
-    my $stage = undef;
-
-    print "* Checking all stages for label $label\n";
-
-    foreach $stage (@stages) {
-
-        # check for increasing faults
-        $faultsNow = $czarDb->countFaultsInPast($label, $stage, "0 MINUTE");
-        $faultsInPast = $czarDb->countFaultsInPast($label, $stage, $interval);
-        if ($faultsNow > $faultsInPast) {
-            $newFaults = $faultsNow - $faultsInPast;
-            print "There have been $newFaults new faults in the last $interval (label='$label', stage='$stage')\n";
-        }
-
-        # check for lack of processing
-        $pendingNow =  $czarDb->countPendingNow($label, $stage);
-        $processedRecently = $czarDb->countProcessed($label, $stage, $interval);
-        if ($pendingNow > 0 && $processedRecently < 1) {
-
-            print "Only $processedRecently exposures have processed out of $pendingNow($faultsNow) pending in the last $interval (label='$label', stage='$stage')\n";
-
-        }
-    }
-}
+} 
 
 ###########################################################################
Index: /branches/eam_branches/ipp-20100823/tools/runcameraexp.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/runcameraexp.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/runcameraexp.pl	(revision 29515)
@@ -0,0 +1,169 @@
+#!/bin/env perl
+
+use strict;
+use warnings;
+use DBI;
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Basename;
+use IO::File;
+
+
+my $dbname = "gpc1";
+my ($cam_id, $update, $redirect, $save_temps, $pretend, $no_verbose);
+
+GetOptions(
+    'cam_id=i'          => \$cam_id,
+    'pretend'           => \$pretend,
+    'redirect-output'   => \$redirect,
+    'save-temps'        => \$save_temps,
+    'update'            => \$update,
+    'dbname=s'          => \$dbname,
+    'no-verbose'        => \$no_verbose, 
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "--cam_id is required", -exitval => 2 ) if !$cam_id;
+
+my $ipprc =  PS::IPP::Config->new();
+my $dbh = getDBHandle();
+
+my $query = "SELECT rawExp.camera, rawExp.exp_tag, camProcessedExp.path_base, camProcessedExp.fault, camRun.dvodb, camRun.reduction, camRun.state FROM camRun JOIN camProcessedExp USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE cam_id = $cam_id";
+
+my $stmt = $dbh->prepare($query);
+$stmt->execute();
+my $results = $stmt->fetchrow_hashref();
+die "query returned no results\n" if !$results;
+my $camera = $results->{camera};
+my $exp_tag = $results->{exp_tag};
+my $path_base = $results->{path_base};
+my $dvodb = $results->{dvodb};
+my $reduction = $results->{reduction};
+my $state = $results->{state};
+my $fault = $results->{fault};
+
+die "path_base not found\n" if !$path_base;
+die "exp_tag not found\n" if !$exp_tag;
+die "dvodb not found\n" if !$dvodb;
+die "camera not found\n" if !$camera;
+die "state not found\n" if !$state;
+die "reduction not found\n" if !$reduction;
+
+die "Cannot update when run is not faulted\n" if $update and ! $fault;
+
+my  $run_state;
+if ($state eq 'full' or $state eq 'new') {
+    $run_state = 'new';
+} elsif ($state eq 'update') {
+    die "unexpected warpRun.state found: $state\n";
+}
+
+my $command = "camera_exp.pl --exp_tag $exp_tag --cam_id $cam_id --camera $camera --outroot $path_base --run-state $run_state ";
+
+$command .= " --reduction $reduction" if $reduction and ($reduction ne "NULL");
+$command .= " --dvodb $dvodb" if $dvodb and ($dvodb ne "NULL");
+$command .= " --redirect-output" if $redirect or $update;
+$command .= " --save-temps" if $save_temps;
+$command .= " --no-update" unless $update;
+$command .= " --verbose" unless $no_verbose;
+$command .= " --dbname $dbname" if $dbname;
+
+
+
+if ($update) {
+    my $revert_command = "camtool -revertprocessedexp -cam_id $cam_id -dbname $dbname";
+    if ($pretend) {
+        print "skipping $revert_command\n";
+    } else {
+        my $rc = system $revert_command;
+        if ($rc != 0) {
+            my $status = $rc >> 8;
+            print STDERR "$revert_command failed: $rc $status\n";
+            exit $status;
+        }
+    }
+}
+print "command to process this camRun\n";
+print "$command\n";
+
+exit 0 if $pretend;
+
+exit system $command;
+
+
+sub getDBHandle {
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+
+    die "database configuration set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+runcameraexp.pl - gather the parameters for and execute camera_exp.pl
+
+=head1 SYNOPSIS
+    
+    perl runcameraexp.pl --cam_id <cam_id> [--update] [--redirect-output] [--pretend] [--dbname <dbname>]
+
+Query the database for the results of a completed camera run and rerun the processing, optionally reverting a faulted
+run.
+
+=over 4
+
+=item * --cam_id <cam_id>
+
+The id of the camera run to process.
+
+=item * --update
+
+Revert a faulted run before processing. If the run is not faulted an error occurs.
+WARNING: insure that the standard science processing either has camera stage turned off or the
+label of this camRun omitted from the label list otherwise it may attempt to processes this run at the same time.
+
+Optional.
+
+=item *  --redirect-output
+
+Send the output of the command to the log file. (This is the default if --update is supplied).
+
+Optional.
+
+=item * --pretend
+
+Just print the commands that would be executed, but do not run them.
+
+Optional.
+
+
+=item * --dbname <dbname>
+
+Name of the IPP databse to query. Default is 'gpc1'.
+
+Optional.
+
+
+=head1 SEE ALSO
+L<runchipimfile.pl>, L<runwarpskyfile.pl>, L<rundiffskyfile.pl>
+
+=cut
Index: /branches/eam_branches/ipp-20100823/tools/rundiffskycell.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/rundiffskycell.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/rundiffskycell.pl	(revision 29515)
@@ -0,0 +1,179 @@
+#!/bin/env perl
+
+# Run diff_skycell.pl for a previously run diff_skyfile.
+# The parameters are obtained from the diffSkyfile table database so it must not be reverted prior
+# to running this script.
+# if the --update argument is supplied we revert the diffSkyfile before running the command.
+# if --update is not supplied we simply re-run the command and leave the database unmodified.
+# WARNING: This script depends on the existing diff.skyfile.run task and diff_skycell.pl.
+# If they are changed this script may need to be modified
+#
+
+use strict;
+use warnings;
+use DBI;
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Basename;
+use IO::File;
+
+my ($first, $last, $stage, $alt_workdir, $no_verbose, $pretend);
+
+my $dbname = "gpc1";
+my ($diff_id, $skycell_id, $threads, $update, $redirect);
+
+GetOptions(
+    'diff_id=i'         => \$diff_id,
+    'skycell_id=s'      => \$skycell_id,
+    'threads=i'         => \$threads,
+    'redirect-output'   => \$redirect,
+    'update'            => \$update,
+    'pretend'           => \$pretend,
+    'dbname=s'          => \$dbname,
+    'no-verbose'        => \$no_verbose, 
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "--diff_id and --skycell_id are required", -exitval => 2 )
+    if !$diff_id or !$skycell_id;
+
+# if we're asked to update redirect the output
+$redirect = 1 if $update;
+
+my $ipprc =  PS::IPP::Config->new();
+my $dbh = getDBHandle();
+
+# find warp warp diffs that have a cleaned magicDSRun
+my $query1 = "SELECT path_base, diff_skyfile_id,bothways,reduction,diff_mode, diffSkyfile.fault FROM diffRun JOIN diffSkyfile USING(diff_id)  JOIN diffInputSkyfile USING(diff_id, skycell_id) WHERE diff_id = $diff_id AND skycell_id = '$skycell_id'";
+
+my $stmt1 = $dbh->prepare($query1);
+$stmt1->execute();
+my $results = $stmt1->fetchrow_hashref();
+my $path_base = $results->{path_base};
+my $diff_skyfile_id = $results->{diff_skyfile_id};
+my $bothways = $results->{bothways};
+my $reduction = $results->{reduction};
+my $fault = $results->{fault};
+
+die "cannot update when skycell is not faulted\n" if $update and !$fault;
+
+die "path_base not found\n" if !$path_base;
+die "diff_skyfile_id\n" if !$path_base;
+
+my $command = "diff_skycell.pl --diff_id $diff_id --skycell_id $skycell_id --diff_skyfile_id $diff_skyfile_id --outroot $path_base --run-state new";
+
+$command .= " --redirect-output" if $redirect;
+$command .= " --no-update" unless $update;
+$command .= " --verbose" unless $no_verbose;
+$command .= " --dbname $dbname" if $dbname;
+$command .= " --inverse" if $bothways;
+$command .= " --reduction $reduction" if $reduction;
+$command .= " --threads $threads" if $threads;
+
+print "command to process this skycell\n";
+print "$command\n";
+
+if ($update) {
+    my $revert_command = "difftool -revertdiffskyfile -diff_id $diff_id -skycell_id $skycell_id -dbname $dbname";
+    if ($pretend) {
+        print "skipping $revert_command\n";
+    } else {
+        my $rc = system $revert_command;
+        if ($rc != 0) {
+            my $status = $rc >> 8;
+            print STDERR "$revert_command failed: $rc $status\n";
+            exit $status;
+        }
+    }
+}
+
+print "command to process this skycell\n";
+print "$command\n";
+
+exit 0 if $pretend;
+
+exit system $command;
+
+
+
+sub getDBHandle {
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+
+    die "database configuration set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+rundiffskyfile.pl - gather the parameters for and execute diff_skycell.pl
+
+=head1 SYNOPSIS
+    
+    perl rundiffskyfile.pl --diff_id <diff_id> --skycell_id <skycell_id> [--threads <num_threads>] [--update] [--redirect-output] [--pretend] [--dbname <dbname>]
+
+Query the database for the results of a diff skycell and rerun the processing, optionally reverting a faulted
+run.
+
+=over 4
+
+=item * --diff_id <diff_id>
+
+The id of the diffSkyfile run to process.
+
+=item * --skycell_id <skycell_id>
+
+The skycell_id of the diffSkyfile run to process.
+
+=item * --threads <num_threads>
+
+The number of threads to use. Default 1.
+Optional.
+
+
+=item * --update
+
+Revert a faulted skycell before processing. If the skycell is not faulted no processing is done.
+WARNING: insure that the standard science procesing either has diff stage turned off or the
+label of this diffRun ommited from the list of labels otherwise it may attempt to processes this run at the same time.
+Optional.
+
+=item *  --redirect-output
+
+Send the output of the command to the logfile. (This is the default if --update is supplied).
+Optional.
+
+=item * --pretend
+
+Just print the commands that would be executed, but do not run them.
+Optional.
+
+=item * --dbname <dbname>
+
+Name of the IPP databse to query. Default is 'gpc1'.
+Optional.
+
+
+=head1 SEE ALSO
+L<runchipimfile.pl>, L<runcameraexp.pl>, L<rundiffskyfile.pl>
+
+=cut
Index: /branches/eam_branches/ipp-20100823/tools/runstackskycell.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/runstackskycell.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/runstackskycell.pl	(revision 29515)
@@ -0,0 +1,169 @@
+#!/bin/env perl
+
+use strict;
+use warnings;
+use DBI;
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Basename;
+use IO::File;
+
+
+my $dbname = "gpc1";
+my ($stack_id, $threads, $update, $redirect, $pretend, $no_verbose);
+
+GetOptions(
+    'stack_id=i'        => \$stack_id,
+    'threads=i'         => \$threads,
+    'pretend'           => \$pretend,
+    'redirect-output'   => \$redirect,
+    'update'            => \$update,
+    'dbname=s'          => \$dbname,
+    'no-verbose'        => \$no_verbose, 
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "--stack_id is required", -exitval => 2 ) if !$stack_id;
+
+my $ipprc =  PS::IPP::Config->new();
+my $dbh = getDBHandle();
+
+my $query = "SELECT * FROM stackRun JOIN stackSumSkyfile USING(stack_id) WHERE stack_id = $stack_id";
+
+my $stmt = $dbh->prepare($query);
+$stmt->execute();
+my $results = $stmt->fetchrow_hashref();
+die "query returned no results\n" if !$results;
+
+my $path_base = $results->{path_base};
+my $tess_id = $results->{tess_id};
+my $reduction = $results->{reduction};
+my $state = $results->{state};
+my $fault = $results->{fault};
+
+die "path_base not found\n" if !$path_base;
+die "tess_id not found\n" if !$tess_id;
+#die "reduction not found\n" if ! defined $reduction;
+die "state not found\n" if !$state;
+die "fault not found\n" if !defined $fault;
+
+die "Cannot update when run is not faulted\n" if $update and ! $fault;
+
+my  $run_state;
+if ($state eq 'full' or $state eq 'new') {
+    $run_state = 'new';
+} elsif ($state eq 'update') {
+    die "unexpected stackRun.state found: $state\n";
+}
+
+my $command = "stack_skycell.pl --stack_id $stack_id --outroot $path_base --run-state $run_state ";
+
+$command .= " --reduction $reduction" if $reduction and ($reduction ne "NULL");
+$command .= " --threads $threads" if $threads;
+$command .= " --redirect-output" if $redirect or $update;
+$command .= " --no-update" unless $update;
+$command .= " --verbose" unless $no_verbose;
+$command .= " --dbname $dbname" if $dbname;
+
+if ($update) {
+    my $revert_command = "stacktool -revertsumskyfile -stack_id $stack_id -dbname $dbname";
+    if ($pretend) {
+        print "skipping $revert_command\n";
+    } else {
+        my $rc = system $revert_command;
+        if ($rc != 0) {
+            my $status = $rc >> 8;
+            print STDERR "$revert_command failed: $rc $status\n";
+            exit $status;
+        }
+    }
+}
+print "command to process this stackRun\n";
+print "$command\n";
+
+exit 0 if $pretend;
+
+exit system $command;
+
+
+sub getDBHandle {
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+
+    die "database configuration set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+runstackskycell.pl - gather the parameters for and execute stack_skycell.pl
+
+=head1 SYNOPSIS
+    
+    perl runstackskycell.pl --stack_id <stack_id> [--update] [--redirect-output] [--pretend] [--threads <nthreads>] [--dbname <dbname>]
+
+Query the database for the results of a completed stack run and rerun the processing, optionally reverting a faulted
+run.
+
+=over 4
+
+=item * --stack_id <stack_id>
+
+The id of the stack run to process.
+
+=item * --update
+
+Revert a faulted run before processing. If the run is not faulted an error occurs.
+WARNING: insure that the standard science processing either has stack stage turned off or the
+label of this stackRun omitted from the label list otherwise it may attempt to processes this run at the same time.
+
+Optional.
+
+=item *  --redirect-output
+
+Send the output of the command to the log file. (This is the default if --update is supplied).
+
+Optional.
+
+=item * --pretend
+
+Just print the commands that would be executed, but do not run them.
+
+Optional.
+
+=item * --threads <nthrads>
+
+Number of threads to be used by use ppStack. Default is 1 (single threaded).
+
+Optional.
+
+=item * --dbname <dbname>
+
+Name of the IPP databse to query. Default is 'gpc1'.
+
+Optional.
+
+
+=head1 SEE ALSO
+L<runchipimfile.pl>, L<runwarpskyfile.pl>, L<rundiffskyfile.pl>
+
+=cut
Index: /branches/eam_branches/ipp-20100823/tools/runwarpskycell.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/runwarpskycell.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/runwarpskycell.pl	(revision 29515)
@@ -0,0 +1,184 @@
+#!/bin/env perl
+
+use strict;
+use warnings;
+use DBI;
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Basename;
+use IO::File;
+
+my ($first, $last, $stage, $alt_workdir, $no_verbose);
+
+my $dbname = "gpc1";
+my ($warp_id, $skycell_id, $threads, $update, $redirect, $pretend, $save_temps);
+
+GetOptions(
+    'warp_id=i'         => \$warp_id,
+    'skycell_id=s'      => \$skycell_id,
+    'threads=i'         => \$threads,
+    'pretend'           => \$pretend,
+    'redirect-output'   => \$redirect,
+    'update'            => \$update,
+    'dbname=s'          => \$dbname,
+    'no-verbose'        => \$no_verbose, 
+    'save-temps'        => \$save_temps,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "--warp_id and --skycell_id are required", -exitval => 2 )
+    if !$warp_id or !$skycell_id;
+
+my $ipprc =  PS::IPP::Config->new();
+my $dbh = getDBHandle();
+
+my $query1 = "SELECT warpSkyfile.path_base, warpSkyfile.fault, warp_skyfile_id, warpRun.tess_id, warpRun.reduction, chipRun.magicked, rawExp.camera, warpRun.state FROM warpRun JOIN warpSkyfile USING(warp_id) JOIN warpImfile USING(warp_id, skycell_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_id = $warp_id AND skycell_id = '$skycell_id'";
+
+my $stmt1 = $dbh->prepare($query1);
+$stmt1->execute();
+my $results = $stmt1->fetchrow_hashref();
+die "query returned no results\n" if !$results;
+my $path_base = $results->{path_base};
+my $warp_skyfile_id = $results->{warp_skyfile_id};
+my $tess_id = $results->{tess_id};
+my $reduction = $results->{reduction};
+my $magicked = $results->{magicked};
+my $camera = $results->{camera};
+my $state = $results->{state};
+my $fault = $results->{fault};
+
+die "cannot update database for a skycell that is not faulted\n" if $update and !$fault;
+
+die "path_base not found\n" if !$path_base;
+die "warp_skyfile_id not found\n" if !$warp_skyfile_id;
+die "tess_id not found\n" if !$tess_id;
+die "magicked not found\n" if ! defined $magicked;
+die "camera not found\n" if !$camera;
+die "state not found\n" if !$state;
+
+my  $run_state;
+if ($state eq 'full' or $state eq 'new') {
+    $run_state = 'new';
+} elsif ($state eq 'update') {
+    $state = 'update';
+    # If the input chipRun has magicked < 0 we do not proceed because we don't know the magicked state of the inputs
+    # The warptool -towarped query gets the right magicked value from the chipProcessedImfiles. Our simple query
+    # does not and I'm not going to bother doint that today.
+    die "chipRun is not fully destreaked. This script does not support this state\n" if $magicked < 0;
+} else {
+    die "unexpected warpRun.state found: $state\n";
+}
+
+my $command = "warp_skycell.pl --warp_id $warp_id --skycell_id $skycell_id --warp_skyfile_id $warp_skyfile_id --outroot $path_base --run-state $run_state --tess_dir $tess_id --camera $camera" ;
+
+$command .= " --magicked $magicked" if $magicked > 0;
+$command .= " --reduction $reduction" if $reduction and ($reduction ne "NULL");
+$command .= " --redirect-output" if $redirect;
+$command .= " --save-temps" if $save_temps;
+$command .= " --no-update" unless $update;
+$command .= " --verbose" unless $no_verbose;
+$command .= " --dbname $dbname" if $dbname;
+
+
+if ($update) {
+    my $revert_command = "warptool -revertwarped -warp_id $warp_id -skycell_id $skycell_id -dbname $dbname";
+    if ($pretend) {
+        print "skipping $revert_command\n";
+    } else {
+        my $rc = system $revert_command;
+        if ($rc != 0) {
+            my $status = $rc >> 8;
+            print STDERR "$revert_command failed: $rc $status\n";
+            exit $status;
+        }
+    }
+}
+print "command to process this skycell\n";
+print "$command\n";
+
+exit 0 if $pretend;
+
+exit system $command;
+
+
+sub getDBHandle {
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+
+    die "database configuration set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+runwarpskyfile.pl - gather the parameters for and execute warp_skycell.pl
+
+=head1 SYNOPSIS
+    
+    perl runwarpskyfile.pl --warp_id <warp_id> --skycell_id <skycell_id> [--threads <num_threads>] [--update] [--redirect-output] [--pretend] [--dbname <dbname>]
+
+Query the database for the results of a warp skycell and rerun the processing, optionally reverting a faulted
+run.
+
+=over 4
+
+=item * --warp_id <warp_id>
+
+The id of the warpSkyfile run to process.
+
+=item * --skycell_id <skycell_id>
+
+The skycell_id of the warpSkyfile run to process.
+
+=item * --threads <num_threads>
+
+The number of threads to use. Default 1.
+Optional.
+
+
+=item * --update
+
+Revert a faulted skycell before processing. If the skycell is not faulted no processing is done.
+WARNING: insure that the standard science procesing either has warp stage turned off or the
+label of this warpRun ommited from the list of labels otherwise it may attempt to processes this run at the same time.
+Optional.
+
+=item *  --redirect-output
+
+Send the output of the command to the logfile. (This is the default if --update is supplied).
+Optional.
+
+=item * --pretend
+
+Just print the commands that would be executed, but do not run them.
+Optional.
+
+=item * --dbname <dbname>
+
+Name of the IPP databse to query. Default is 'gpc1'.
+Optional.
+
+
+=head1 SEE ALSO
+L<runchipimfile.pl>, L<runcameraexp.pl>, L<rundiffskyfile.pl>
+
+=cut
Index: /branches/eam_branches/ipp-20100823/tools/shuffle_otas.pl
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/shuffle_otas.pl	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/shuffle_otas.pl	(revision 29515)
@@ -0,0 +1,148 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+# Step 1: Get hostname
+use Sys::Hostname;
+my $host = hostname();
+
+use IPC::Cmd 0.36 qw( can_run run );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use DBI;
+
+my $missing_tools;
+my $nebshift = can_run('neb-shift') or (warn "Can't find neb-shift" and $missing_tools = 1);
+my ($destination,$so_id_start,$so_id_end,$limit,$dbname,$verbose,$no_pretend);
+$limit = 10;
+$so_id_start = 0;
+$so_id_end = 0;
+GetOptions(
+    'host=s'         => \$host,
+    'destination|d=s' => \$destination,
+    'so_id_start=s'    => \$so_id_start,
+    'so_id_end=s'      => \$so_id_end,
+    'limit=s'        => \$limit,
+    'dbname=s'       => \$dbname,
+    'verbose'        => \$verbose,
+    'no_pretend'     => \$no_pretend,
+    ) or pod2usage ( 2 );
+pod2usage( -msg =>
+"USAGE: shuffle_otas.pl <options>
+        Options:
+           --host <host>          Host to take from
+           --destination <host>   Host to dump to
+           --limit <N>            Number to move attempt to move
+           --so_id_start <SO_ID>  so_id to start with
+           --so_id_end   <SO_ID>  so_id to end with
+           --dbname <db>          Database name
+           --verbose              Print status checks
+           --no_pretend           Actually move files.\n", -exitval => 2) 
+
+    unless (defined $destination and defined $dbname);
+
+
+# Step 2: Convert hostname into vol_id and check for availability
+# Make configurable
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+my $dbserver = 'ippdb00';
+my $dbuser = 'ipp';
+my $dbpass = 'ipp';
+my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
+		   "mysql_socket=" . DB_SOCKET(),
+		   ${dbuser},${dbpass},
+		   { RaiseError => 1, AutoCommit => 1}
+    ) or die "Unable to connect to database $DBI::errstr\n";
+
+# This should be a nebulous function
+my $sth_get_vol_id = "SELECT vol_id,volume.name,cab_id FROM mountedvol JOIN volume USING(vol_id) WHERE mountedvol.available = 1 AND mountedvol.allocate = 1 AND mountedvol.host = '$host'";
+
+my $dr_get_vol_id = $db->selectall_arrayref( $sth_get_vol_id );
+if ($#{ $dr_get_vol_id } > 0) {
+    die "Too many volumes returned\n";
+}
+my $vol_id = shift(@{ ${ $dr_get_vol_id }[0] });
+my $vol_name = shift(@{ ${ $dr_get_vol_id }[0] });
+
+$sth_get_vol_id = "SELECT vol_id,volume.name,cab_id FROM mountedvol JOIN volume USING(vol_id) WHERE mountedvol.available = 1 AND mountedvol.allocate = 1 AND mountedvol.host = '$destination'";
+
+$dr_get_vol_id = $db->selectall_arrayref( $sth_get_vol_id );
+if ($#{ $dr_get_vol_id } > 0) {
+    die "Too many volumes returned\n";
+}
+shift(@{ ${ $dr_get_vol_id }[0] });
+my $destination_name = shift(@{ ${ $dr_get_vol_id }[0] });
+my $forbidden_cab_id = shift(@{ ${ $dr_get_vol_id }[0] });
+
+if ($verbose) {
+    print STDERR "Got vol_id $vol_id vol_name $vol_name cab_id $forbidden_cab_id\n";
+}
+# Step 3: Get a list of replicated items with one instance on this host
+# select * from instance JOIN storage_object USING(so_id) JOIN storage_object_xattr USING(so_id) where vol_id = 26 AND name = 'user.copies' AND value >= 2 limit 10;
+# Step 4: Identify which are primary and which are secondary copies ...
+# select * FROM (select K.vol_id AS here,K.ins_id,K.so_id,ext_id,value,II,instance.vol_id AS there from (select V.*,MAX(instance.ins_id) AS II from (select vol_id,ins_id,so_id,ext_id,value from storage_object JOIN storage_object_xattr USING(so_id) JOIN instance USING(so_id) where vol_id = 26 AND name = 'user.copies' AND value >= 2 limit 100) AS V LEFT OUTER JOIN instance USING(so_id) GROUP BY so_id) AS K JOIN instance ON II = instance.ins_id GROUP BY so_id) AS T WHERE here = there;
+
+my $sth_get_ext_ids = " SELECT * FROM ( ";
+$sth_get_ext_ids .= " SELECT K.vol_id AS here,K.ins_id,K.so_id,ext_id,value, ";
+$sth_get_ext_ids .= " MAXins_id,instance.vol_id AS there FROM ( ";
+$sth_get_ext_ids .= " SELECT V.*,MAX(instance.ins_id) AS MAXins_id FROM ( ";
+$sth_get_ext_ids .= " SELECT vol_id,ins_id,so_id,ext_id,value FROM ";
+$sth_get_ext_ids .= " storage_object JOIN storage_object_xattr USING(so_id) ";
+$sth_get_ext_ids .= " JOIN instance USING(so_id) ";
+$sth_get_ext_ids .= "  WHERE vol_id = $vol_id AND name = 'user.copies' AND value >= 2 ";
+if ($so_id_start > 0) {
+    $sth_get_ext_ids .= " AND so_id > $so_id_start ";
+}
+if ($so_id_end > 0) {
+    $sth_get_ext_ids .= " AND so_id < $so_id_end ";
+}
+if ($limit > 0) {
+    $sth_get_ext_ids .= " limit $limit ";
+}
+$sth_get_ext_ids .= " ) AS V ";
+$sth_get_ext_ids .= " LEFT OUTER JOIN instance USING(so_id) GROUP BY so_id ) AS K ";
+$sth_get_ext_ids .= " JOIN instance on MAXins_id = instance.ins_id GROUP BY so_id ) AS T ";
+$sth_get_ext_ids .= " where here = there ";
+
+if ($verbose) {
+    print STDERR "$sth_get_ext_ids\n";
+}
+my $dr_get_ext_ids = $db->selectall_arrayref( $sth_get_ext_ids );
+my $counter = 0;
+foreach my $row (@{ $dr_get_ext_ids }) {
+    $counter++;
+    my ($current_vol_id,$current_ins_id,$so_id,$ext_id,$count,
+	$max_ins_id,$check_vol_id) = @{ $row };
+    if (($current_vol_id != $check_vol_id)||($count < 2)) {
+	warn "Skipping row for $ext_id due to impossible situation.";
+    }
+    if ($current_vol_id == $vol_id) {
+	my $sth_get_excludes = "SELECT MINins_id,vol_id,cab_id FROM (SELECT MIN(ins_id) AS MINins_id FROM instance WHERE so_id = $so_id GROUP BY so_id) AS I JOIN instance ON ins_id = MINins_id JOIN volume USING(vol_id) WHERE cab_id != $forbidden_cab_id";
+	my $dr_get_excludes = $db->selectall_arrayref( $sth_get_excludes );
+	if ($#{ $dr_get_excludes } >= 0) {
+	    my ($min_ins_id,$ok_vol_id,$ok_cab_id) = @{ ${ $dr_get_excludes }[0] };
+	    
+	    if ($ok_cab_id != $forbidden_cab_id) {
+		if ($verbose) {
+		    print STDERR "Acceptable to move because ($max_ins_id,$current_ins_id,$current_vol_id,$forbidden_cab_id) != ($min_ins_id,$ok_vol_id,$ok_cab_id) Count: $counter\n";
+		}
+		my $cmd = "$nebshift --volume $destination_name $ext_id $vol_name";
+		print "$cmd\n";
+		if ($no_pretend) {
+		    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+			run(command => $cmd, verbose => $verbose);
+		    unless ($success) {
+			$error_code = (($error_code >> 8) or 4);
+			die("Unable to perform nebshift: $error_code $ext_id\n");
+		    }
+		}		    
+	    }
+	}	
+    }
+}
+
+
+# Step 5: Call neb-shift to move file around.
+# neb-shift --volume ipp00[5-7] $ext_id $hostname
Index: /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh.record
===================================================================
--- /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh.record	(revision 29515)
+++ /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh.record	(revision 29515)
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+INSTALLDIR=/home/panstarrs/ipp/who_uses_the_cluster
+$INSTALLDIR/_who_uses_the_cluster.sh
+echo "Sleeping 17s"
+sleep 17
+CLUSTERMONITORDIR=~/htdocs/clusterMonitor
+mkdir -p $CLUSTERMONITORDIR
+WHEN=`date +"%s"`
+/bin/cp -f $CLUSTERMONITORDIR/top.html /home/panstarrs/ipp/cluster_use/top.$WHEN.html
+/bin/gzip /home/panstarrs/ipp/cluster_use/top.$WHEN.html
+/bin/rm -f $CLUSTERMONITORDIR/top.html
+$INSTALLDIR/_who_uses_the_cluster_analyze.pl > $CLUSTERMONITORDIR/top.html