Index: branches/bills_081204/Ohana/src/addstar/doc/notes.txt
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/addstar/doc/notes.txt	(revision 20530)
+++ branches/bills_081204/Ohana/src/addstar/doc/notes.txt	(revision 20890)
@@ -1,2 +1,21 @@
+
+2008.10.11
+
+  updating addstar / dvo to deal better with mosaic / image relationships
+
+  * fix imageID sequencing (DONE)
+  * define imageID -> image lookup
+  * add parentID (for mosaic relationship)
+  * add external ID
+  * add supplied filename (different from 'filename')
+
+  there are two issues involved in mosaic astrometry:  1) we need to
+  have links in the Image table between the chip-level images and
+  their associated mosaic images.  this is provided by the parentID
+  field.  2) the coord transformation functions use a local static
+  variable to supply the associated mosaic coords structure.  this is
+  a weak implementation, but the method for now.
+
+  
 
 2008.02.23
Index: branches/bills_081204/Ohana/src/addstar/src/GetFileMode.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/addstar/src/GetFileMode.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/addstar/src/GetFileMode.c	(revision 20890)
@@ -14,12 +14,13 @@
   gfits_scan (header, "EXTEND", "%t", 1, &extend);
     
-  { 
-    int tmp, haveCAMCOL, haveSTRIPE;
-    
-    // SDSS tsObj files have CAMCOL & STRIP keywords present in the header
-    haveCAMCOL = gfits_scan (header, "CAMCOL",  "%d", 1, &tmp);
-    haveSTRIPE = gfits_scan (header, "STRIPE",  "%d", 1, &tmp);
-    if (haveCAMCOL && haveSTRIPE) return SDSS_OBJ;
-  }
+{
+    int tmp, havePHOT_VER, haveTARG_VER;
+
+    // SDSS tsObj files have a version number for the PHOTO and 
+    // TS (target selection) pipelines present as header keywords
+    havePHOT_VER = gfits_scan (header, "PHOT_VER", "%s", 1, &tmp);
+    haveTARG_VER = gfits_scan (header, "TARG_VER", "%s", 1, &tmp);
+    if (havePHOT_VER && haveTARG_VER) return SDSS_OBJ;
+}
 
   if ((Naxis == 2) || TEXTMODE || !simple) {
Index: branches/bills_081204/Ohana/src/addstar/src/ReadStarsSDSS.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/addstar/src/ReadStarsSDSS.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/addstar/src/ReadStarsSDSS.c	(revision 20890)
@@ -71,4 +71,9 @@
   NAMED_PHOTCODE_AND_ZP (photcode[3], zeropt[3], "I_SDSS");
   NAMED_PHOTCODE_AND_ZP (photcode[4], zeropt[4], "Z_SDSS");
+
+  // XXXYYYZZZ SDSS tables have special flags for undefined/unmeasured values and errors:
+  // -9999 flags unmeasured values, and the corresponding error may or may not be meaningful
+  // -1000 flags errors that are not determined, even though the corresponding quantity is.
+  // These special values need to be trapped here to avoid averaging meaningful numbers with the flag values.
 
   // various header values needed to calculate per-star data below
Index: branches/bills_081204/Ohana/src/getstar/src/getstar.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/getstar/src/getstar.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/getstar/src/getstar.c	(revision 20890)
@@ -26,6 +26,21 @@
   // this has to be here because the 'open' inits the catalog (perhaps not ideal)
   if (!strcmp (OUTFORMAT, "CATALOG")) {
-    unlink (OUTPUT);
-    dvo_catalog_open   (&output, NULL, VERBOSE, "w");
+      int length;
+      char *filename, *path, *root;
+      // we need to unlink the files, otherwise dvo_catalog_open ("w") will error if they exist
+      unlink (output.filename);
+      if (output.catmode == DVO_MODE_SPLIT) {
+	  path = pathname (output.filename);
+	  root = filerootname (output.filename);
+	  length = strlen(path) + strlen(root) + 6;
+	  ALLOCATE (filename, char, length);
+	  sprintf (filename, "%s/%s.cpm", path, root);
+	  unlink (filename);
+	  sprintf (filename, "%s/%s.cpn", path, root);
+	  unlink (filename);
+	  sprintf (filename, "%s/%s.cps", path, root);
+	  unlink (filename);
+      }
+      dvo_catalog_open   (&output, NULL, VERBOSE, "w");
   }
 
Index: branches/bills_081204/Ohana/src/gophot/src/gophot.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/gophot/src/gophot.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/gophot/src/gophot.c	(revision 20890)
@@ -25,5 +25,5 @@
   }
   /* convert to float, set up noise array and axes */
-  gfits_convert_format (&header, &matrix, -32, 1.0, 0.0, 0);
+  gfits_convert_format (&header, &matrix, -32, 1.0, 0.0, 0xffff, 0);
   ALLOCATE (noise, float, matrix.size);
   big = (float *) matrix.buffer;
Index: branches/bills_081204/Ohana/src/libfits/include/gfitsio.h
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/libfits/include/gfitsio.h	(revision 20530)
+++ branches/bills_081204/Ohana/src/libfits/include/gfitsio.h	(revision 20890)
@@ -138,5 +138,5 @@
 
 void   	gfits_add_matrix_value         PROTO((Matrix *matrix, int x, int y, double value)); 
-int    	gfits_convert_format           PROTO((Header *header, Matrix *matrix, int outBitpix, double outScale, double outZero, int outUnsign));
+int    	gfits_convert_format           PROTO((Header *header, Matrix *matrix, int outBitpix, double outScale, double outZero, int inBlank, int outUnsign));
 int     gfits_copy_matrix              PROTO((Matrix *in, Matrix *out)); 
 int     gfits_create_matrix            PROTO((Header *header, Matrix *matrix)); 
@@ -157,5 +157,5 @@
 int     gfits_uncompress_image 	       PROTO((Header *header, Matrix *matrix, FTable *ftable));
 int     gfits_uncompress_data  	       PROTO((char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize));
-int     gfits_distribute_data  	       PROTO((Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int *ztile, float zscale, float zzero));
+int     gfits_distribute_data  	       PROTO((Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero));
 int     gfits_byteswap_zdata   	       PROTO((char *zdata, int Nzdata, int bitpix));
 int     gfits_extension_is_compressed  PROTO((Header *header));
Index: branches/bills_081204/Ohana/src/libfits/matrix/F_compress_M.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/libfits/matrix/F_compress_M.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/libfits/matrix/F_compress_M.c	(revision 20890)
@@ -34,5 +34,5 @@
   char cmptype[80];
   char zaxis[10], naxis[10], key[10], word[81], exttype[81], checksum[81], datasum[81];
-  int Noptions, NOPTIONS;
+  int Noptions, NOPTIONS, zblank, oblank;
   VarLengthColumn zdef;
   float zscale, zzero;
@@ -177,4 +177,10 @@
   gfits_scan (header, "ZSCALE", "%lf", 1, &zscale);
 
+  zblank = 32767;
+  gfits_scan (header, "ZBLANK", "%d", 1, &zblank);
+
+  oblank = 32767;
+  gfits_scan (header, "BLANK", "%d", 1, &oblank);
+
   zzero = 0;
   gfits_scan (header, "ZZERO", "%lf", 1, &zzero);
@@ -248,5 +254,5 @@
 
     // copy the uncompressed pixels into their correct locations 	    
-    if (!gfits_distribute_data (matrix, odata_pixsize, out, Nout, otile, ztile, zscale, zzero)) return (FALSE);
+    if (!gfits_distribute_data (matrix, odata_pixsize, out, Nout, otile, oblank, ztile, zblank, zscale, zzero)) return (FALSE);
 
     // update the tile counters, carrying to the next dimension if needed
@@ -289,5 +295,5 @@
 
 // bitpix is the input data size/type
-int gfits_distribute_data (Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int *ztile, float zscale, float zzero) {
+int gfits_distribute_data (Matrix *matrix, int bitpix, char *data, int Ndata, int *otile, int oblank, int *ztile, int zblank, float zscale, float zzero) {
 
   int i, j, start, offset, coord, Nline;
@@ -331,6 +337,9 @@
     TYPE *Optr = (TYPE *) &matrix->buffer[SIZE*(offset + start)]; \
     for (j = 0; j < Ztile[0]; j++, Iptr++, Optr++) { \
-      *Optr = *Iptr * zscale + zzero; \
-    } }
+      if (*Iptr == zblank) { \
+	  *Optr = oblank;    \
+      } else { \
+	  *Optr = *Iptr * zscale + zzero;	\
+      } } }
 
   // this macro sets up the outer switch and calls above macro with all output bitpix options
Index: branches/bills_081204/Ohana/src/libfits/matrix/F_convert_format.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/libfits/matrix/F_convert_format.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/libfits/matrix/F_convert_format.c	(revision 20890)
@@ -2,32 +2,93 @@
 # include <gfitsio.h>
 
-# define CONVERTDOWN \
-  inMode  *in; \
-  outMode *out; \
-  out = (outMode *) matrix[0].buffer; \
-  in  = (inMode  *) matrix[0].buffer; \
-  for (i = 0; i < Npixels; i++, out++, in++) \
-    *out = *in*A + B; \
+# ifndef NAN
+# ifndef BYTE_SWAP
+#  define __nan_bytes           { 0x7f, 0xc0, 0, 0 }
+# else
+#  define __nan_bytes           { 0, 0, 0xc0, 0x7f }
+# endif
+static union { unsigned char __c[4]; float __d; } __nan_union
+__attribute_used__ = { __nan_bytes };
+# define NAN    (__nan_union.__d)
+# endif
+
+# define CONVERTDOWN(MY_NAN)				\
+  inMode  *in;						\
+  outMode *out;						\
+  out = (outMode *) matrix[0].buffer;			\
+  in  = (inMode  *) matrix[0].buffer;			\
+  for (i = 0; i < Npixels; i++, out++, in++)		\
+    if (*in == inBlank) {				\
+      *out = (MY_NAN);					\
+    } else {						\
+      *out = *in*A + B;					\
+    }							\
   REALLOCATE (matrix[0].buffer, char, matrix[0].size); 
 
-# define CONVERTSAME \
-  inMode  *in; \
-  outMode *out; \
-  out = (outMode *) matrix[0].buffer; \
-  in  = (inMode  *) matrix[0].buffer; \
-  for (i = 0; i < Npixels; i++, out++, in++) \
-    *out = *in*A + B; 
-
-# define CONVERTUP \
-  inMode  *in; \
-  outMode *out; \
-  REALLOCATE (matrix[0].buffer, char, matrix[0].size); \
-  out = (outMode *)matrix[0].buffer + Npixels - 1; \
-  in  = (inMode  *)matrix[0].buffer + Npixels - 1; \
-  for (i = 0; i < Npixels; i++, out--, in--) \
-    *out = *in*A + B;
+# define CONVERTSAME(MY_NAN)			\
+  inMode  *in;					\
+  outMode *out;					\
+  out = (outMode *) matrix[0].buffer;		\
+  in  = (inMode  *) matrix[0].buffer;		\
+  for (i = 0; i < Npixels; i++, out++, in++)	\
+    if (*in == inBlank) {			\
+      *out = (MY_NAN);				\
+    } else {					\
+      *out = *in*A + B;				\
+    }
+
+# define CONVERTUP(MY_NAN)				\
+  inMode  *in;						\
+  outMode *out;						\
+  REALLOCATE (matrix[0].buffer, char, matrix[0].size);	\
+  out = (outMode *)matrix[0].buffer + Npixels - 1;	\
+  in  = (inMode  *)matrix[0].buffer + Npixels - 1;	\
+  for (i = 0; i < Npixels; i++, out--, in--)		\
+    if (*in == inBlank) {				\
+      *out = (MY_NAN);					\
+    } else {						\
+      *out = *in*A + B;					\
+    }
+
+# define CONVERTDOWN_FF(MY_NAN)				\
+  inMode  *in;						\
+  outMode *out;						\
+  out = (outMode *) matrix[0].buffer;			\
+  in  = (inMode  *) matrix[0].buffer;			\
+  for (i = 0; i < Npixels; i++, out++, in++)		\
+    if (isnan(*in) || isinf(*in)) {			\
+      *out = (MY_NAN);					\
+    } else {						\
+      *out = *in*A + B;					\
+    }							\
+  REALLOCATE (matrix[0].buffer, char, matrix[0].size); 
+
+# define CONVERTSAME_FF(MY_NAN)			\
+  inMode  *in;					\
+  outMode *out;					\
+  out = (outMode *) matrix[0].buffer;		\
+  in  = (inMode  *) matrix[0].buffer;		\
+  for (i = 0; i < Npixels; i++, out++, in++)	\
+    if (isnan(*in) || isinf(*in)) {			\
+      *out = (MY_NAN);				\
+    } else {					\
+      *out = *in*A + B;				\
+    }
+
+# define CONVERTUP_FF(MY_NAN)				\
+  inMode  *in;						\
+  outMode *out;						\
+  REALLOCATE (matrix[0].buffer, char, matrix[0].size);	\
+  out = (outMode *)matrix[0].buffer + Npixels - 1;	\
+  in  = (inMode  *)matrix[0].buffer + Npixels - 1;	\
+  for (i = 0; i < Npixels; i++, out--, in--)		\
+    if (isnan(*in) || isinf(*in)) {			\
+      *out = (MY_NAN);					\
+    } else {						\
+      *out = *in*A + B;					\
+    }
 
 /*********************** fits convert format ***********************************/
-int gfits_convert_format (Header *header, Matrix *matrix, int outBitpix, double outScale, double outZero, int outUnsign) {
+int gfits_convert_format (Header *header, Matrix *matrix, int outBitpix, double outScale, double outZero, int inBlank, int outUnsign) {
 
   unsigned long i, nbytes, Npixels;
@@ -63,300 +124,300 @@
   if ((!outUnsign) && (!inUnsign)) {  /** BLOCK 1 **/
     switch (inBitpix) {
-    case 8: 
-      { typedef unsigned char inMode;
+      case 8: 
+	{ typedef unsigned char inMode;
 	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTSAME; break; }
-	case 16:  { typedef short          outMode; CONVERTUP;   break; }
-	case -16: { typedef unsigned short outMode; CONVERTUP;   break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 16: 
-      { typedef short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short outMode; CONVERTSAME; break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case -16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short outMode; CONVERTSAME; break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 32: 
-      { typedef int inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTSAME; break; }
-	case -32: { typedef float          outMode; CONVERTSAME; break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -32: 
-      { typedef float inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTSAME; break; }
-	case -32: { typedef float          outMode; CONVERTSAME; break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -64: 
-      { typedef double inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTDOWN; break; }
-	case -32: { typedef float          outMode; CONVERTDOWN; break; }
-	case -64: { typedef double         outMode; CONVERTSAME; break; }
-	}
-      }
+	  case 8:   { typedef unsigned char  outMode; CONVERTSAME(0); break; }
+	  case 16:  { typedef short          outMode; CONVERTUP(0);   break; }
+	  case -16: { typedef unsigned short outMode; CONVERTUP(0);   break; }
+	  case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	  case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	  case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	}
+	}
+	break;  
+      case 16: 
+	{ typedef short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case -16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case 32: 
+	{ typedef int inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTSAME(0); break; }
+	    case -32: { typedef float          outMode; CONVERTSAME(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;
+      case -32: 
+	{ typedef float inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTSAME_FF(0); break; }
+	    case -32: { typedef float          outMode; CONVERTSAME_FF(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTUP_FF(NAN);   break; }
+	  }
+	}
+	break;
+      case -64: 
+	{ typedef double inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTDOWN_FF(0); break; }
+	    case -32: { typedef float          outMode; CONVERTDOWN_FF(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTSAME_FF(NAN); break; }
+	  }
+	}
     }
   }
   if ((outUnsign) && (!inUnsign)) {  /** BLOCK 3 **/
     switch (inBitpix) {
-    case 8: 
-      { typedef unsigned char inMode;
+      case 8: 
+	{ typedef unsigned char inMode;
 	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTSAME; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTUP;   break; }
-	case -16: { typedef unsigned short  outMode; CONVERTUP;   break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 16: 
-      { typedef short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case -16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 32: 
-      { typedef int inMode;
-	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTSAME; break; }
-	case -32: { typedef          float  outMode; CONVERTSAME; break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -32: 
-      { typedef float inMode;
-	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTSAME; break; }
-	case -32: { typedef          float  outMode; CONVERTSAME; break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -64: 
-      { typedef double inMode;
-	switch (outBitpix) {
-	case 8:   { typedef          char   outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTDOWN; break; }
-	case -32: { typedef          float  outMode; CONVERTDOWN; break; }
-	case -64: { typedef          double outMode; CONVERTSAME; break; }
-	}
-      }
+	  case 8:   { typedef          char   outMode; CONVERTSAME(0); break; }
+	  case 16:  { typedef unsigned short  outMode; CONVERTUP(0);   break; }
+	  case -16: { typedef unsigned short  outMode; CONVERTUP(0);   break; }
+	  case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	  case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	  case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	}
+	}
+	break;  
+      case 16: 
+	{ typedef short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef          char   outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case -16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef          char   outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case 32: 
+	{ typedef int inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef          char   outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTSAME(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTSAME(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;
+      case -32: 
+	{ typedef float inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef          char   outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTSAME_FF(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTSAME_FF(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTUP_FF(NAN);   break; }
+	  }
+	}
+	break;
+      case -64: 
+	{ typedef double inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef          char   outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTDOWN_FF(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTDOWN_FF(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTSAME_FF(NAN); break; }
+	  }
+	}
     }
   }
   if ((!outUnsign) && (inUnsign)) {  /** BLOCK 2 **/
     switch (inBitpix) {
-    case 8: 
-      { typedef unsigned char inMode;
+      case 8: 
+	{ typedef unsigned char inMode;
 	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTSAME; break; }
-	case 16:  { typedef short          outMode; CONVERTUP;   break; }
-	case -16: { typedef unsigned short outMode; CONVERTUP;   break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short outMode; CONVERTSAME; break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case -16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short outMode; CONVERTSAME; break; }
-	case 32:  { typedef int            outMode; CONVERTUP;   break; }
-	case -32: { typedef float          outMode; CONVERTUP;   break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 32: 
-      { typedef unsigned int inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTSAME; break; }
-	case -32: { typedef float          outMode; CONVERTSAME; break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -32: 
-      { typedef float inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTSAME; break; }
-	case -32: { typedef float          outMode; CONVERTSAME; break; }
-	case -64: { typedef double         outMode; CONVERTUP;   break; }
-	}
-      }
-      break;
-    case -64: 
-      { typedef double inMode;
-	switch (outBitpix) {
-	case 8:   { typedef unsigned char  outMode; CONVERTDOWN; break; }
-	case 16:  { typedef short          outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short outMode; CONVERTDOWN; break; }
-	case 32:  { typedef int            outMode; CONVERTDOWN; break; }
-	case -32: { typedef float          outMode; CONVERTDOWN; break; }
-	case -64: { typedef double         outMode; CONVERTSAME; break; }
-	}
-      }
+	  case 8:   { typedef unsigned char  outMode; CONVERTSAME(0); break; }
+	  case 16:  { typedef short          outMode; CONVERTUP(0);   break; }
+	  case -16: { typedef unsigned short outMode; CONVERTUP(0);   break; }
+	  case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	  case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	  case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	}
+	}
+	break;  
+      case 16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case -16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTSAME(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef float          outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case 32: 
+	{ typedef unsigned int inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTSAME(0); break; }
+	    case -32: { typedef float          outMode; CONVERTSAME(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;
+      case -32: 
+	{ typedef float inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTSAME_FF(0); break; }
+	    case -32: { typedef float          outMode; CONVERTSAME_FF(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTUP_FF(NAN);   break; }
+	  }
+	}
+	break;
+      case -64: 
+	{ typedef double inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef unsigned char  outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef short          outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef int            outMode; CONVERTDOWN_FF(0); break; }
+	    case -32: { typedef float          outMode; CONVERTDOWN_FF(NAN); break; }
+	    case -64: { typedef double         outMode; CONVERTSAME_FF(NAN); break; }
+	  }
+	}
     }
   }
   if ((outUnsign) && (inUnsign)) {
     switch (inBitpix) {
-    case 8: 
-      { typedef char inMode;
+      case 8: 
+	{ typedef char inMode;
 	switch (outBitpix) {
-	case 8:   { typedef char            outMode; CONVERTSAME; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTUP;   break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef char            outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case -16: 
-      { typedef unsigned short inMode;
-	switch (outBitpix) {
-	case 8:   { typedef char            outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTSAME; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTUP;   break; }
-	case -32: { typedef          float  outMode; CONVERTUP;   break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}
-      }
-      break;  
-    case 32: 
-      { typedef unsigned int inMode;
-	switch (outBitpix) {
-	case 8:   { typedef char            outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTSAME; break; }
-	case -32: { typedef          float  outMode; CONVERTSAME; break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}				    
-      }					    
-      break;				    
-    case -32: 				    
-      { typedef float inMode;		    
-	switch (outBitpix) {		    
-	case 8:   { typedef char            outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTSAME; break; }
-	case -32: { typedef          float  outMode; CONVERTSAME; break; }
-	case -64: { typedef          double outMode; CONVERTUP;   break; }
-	}				    
-      }					    
-      break;				    
-    case -64: 				    
-      { typedef double inMode;		    
-	switch (outBitpix) {		    
-	case 8:   { typedef char            outMode; CONVERTDOWN; break; }
-	case 16:  { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case -16: { typedef unsigned short  outMode; CONVERTDOWN; break; }
-	case 32:  { typedef unsigned int    outMode; CONVERTDOWN; break; }
-	case -32: { typedef          float  outMode; CONVERTDOWN; break; }
-	case -64: { typedef          double outMode; CONVERTSAME; break; }
-	}
-      }
+	  case 8:   { typedef char            outMode; CONVERTSAME(0); break; }
+	  case 16:  { typedef unsigned short  outMode; CONVERTUP(0);   break; }
+	  case -16: { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	  case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	  case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	  case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	}
+	}
+	break;  
+      case 16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef char            outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case -16: 
+	{ typedef unsigned short inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef char            outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTSAME(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTUP(0);   break; }
+	    case -32: { typedef          float  outMode; CONVERTUP(NAN);   break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }
+	}
+	break;  
+      case 32: 
+	{ typedef unsigned int inMode;
+	  switch (outBitpix) {
+	    case 8:   { typedef char            outMode; CONVERTDOWN(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTSAME(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTSAME(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTUP(NAN);   break; }
+	  }				    
+	}					    
+	break;				    
+      case -32: 				    
+	{ typedef float inMode;		    
+	  switch (outBitpix) {		    
+	    case 8:   { typedef char            outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTSAME_FF(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTSAME_FF(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTUP_FF(NAN);   break; }
+	  }				    
+	}					    
+	break;				    
+      case -64: 				    
+	{ typedef double inMode;		    
+	  switch (outBitpix) {		    
+	    case 8:   { typedef char            outMode; CONVERTDOWN_FF(0); break; }
+	    case 16:  { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case -16: { typedef unsigned short  outMode; CONVERTDOWN_FF(0); break; }
+	    case 32:  { typedef unsigned int    outMode; CONVERTDOWN_FF(0); break; }
+	    case -32: { typedef          float  outMode; CONVERTDOWN_FF(NAN); break; }
+	    case -64: { typedef          double outMode; CONVERTSAME_FF(NAN); break; }
+	  }
+	}
     }
   }
Index: branches/bills_081204/Ohana/src/opihi/cmd.data/rd.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/cmd.data/rd.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/cmd.data/rd.c	(revision 20890)
@@ -4,5 +4,5 @@
 int rd (int argc, char **argv) {
   
-  int i, N, status, plane, Nplane, extend, Nextend, Nskip, JustHead;
+  int i, N, status, plane, Nplane, extend, Nextend, Nskip, JustHead, blank;
   int ccdsel, done, Nword, IsCompressed;
   char region[512], *ccdid, *filename;
@@ -221,10 +221,14 @@
   buf[0].bzero  = buf[0].header.bzero;     /* store the original values */
   buf[0].unsign = buf[0].header.unsign;
+
   gprint (GP_LOG, "read %d bytes from %s into buffer %s\n", 
 	   buf[0].header.size + buf[0].matrix.size, argv[2], argv[1]);
 
+  blank = 0xffff;
+  gfits_scan (&buf[0].header, "BLANK", "%d", 1, &blank);
+
   /** now - convert the matrix values to floats for internal use **/
-  gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, gfits_get_unsign_mode());
-  
+  gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, blank, gfits_get_unsign_mode());
+
   return (TRUE);
 }
Index: branches/bills_081204/Ohana/src/opihi/cmd.data/rdseg.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/cmd.data/rdseg.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/cmd.data/rdseg.c	(revision 20890)
@@ -4,5 +4,5 @@
 int rdseg (int argc, char **argv) {
   
-  int x, y, nx, ny, status;
+  int x, y, nx, ny, status, blank;
   char region[512], *filename;
   FILE *f;
@@ -65,6 +65,8 @@
 	   buf[0].header.size + buf[0].matrix.size, argv[2], argv[1]);
 
+  gfits_scan (&buf[0].header, "BLANK", "%d", 1, &blank);
+
   /** now - convert the matrix values to floats for internal use **/
-  gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, gfits_get_unsign_mode());
+  gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, blank, gfits_get_unsign_mode());
   
   return (TRUE);
Index: branches/bills_081204/Ohana/src/opihi/cmd.data/stats.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/cmd.data/stats.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/cmd.data/stats.c	(revision 20890)
@@ -57,9 +57,12 @@
   }
 
-  max = min = *((float *)(buf[0].matrix.buffer) + sy*buf[0].matrix.Naxis[0] + sx);
+  max = -1e32;
+  min = +1e32;
   for (j = sy; j < sy + ny; j++) {
     V = (float *)(buf[0].matrix.buffer) + j*buf[0].matrix.Naxis[0] + sx; 
     for (i = 0; i < nx; i++, V++) {
       if (Ignore && (fabs (*V - IgnoreValue) < 1e-8)) continue;
+      if (isnan(*V)) continue;
+      if (isinf(*V)) continue;
       N1 += *V;
       N2 += (*V)*(*V);
@@ -81,4 +84,6 @@
       for (i = 0; i < nx; i++, V++) {
 	if (Ignore && (fabs (*V - IgnoreValue) < 1e-8)) continue;
+	if (isnan(*V)) continue;
+	if (isinf(*V)) continue;
 	bin = MIN (MAX (0, (*V - min) * range), 0xffff);
 	hist[bin] ++;
Index: branches/bills_081204/Ohana/src/opihi/cmd.data/wd.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/cmd.data/wd.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/cmd.data/wd.c	(revision 20890)
@@ -82,5 +82,6 @@
 
   if (temp_header.Naxes) {
-    gfits_convert_format (&temp_header, &temp_matrix, outBitpix, outScale, outZero, outUnsign);
+    // the inBlank value probably does not matter: temp_matrix is float, so nan is used
+    gfits_convert_format (&temp_header, &temp_matrix, outBitpix, outScale, outZero, 0xffff, outUnsign);
   } else {
     gfits_modify (&temp_header, "BITPIX", "%d", 1, outBitpix);
Index: branches/bills_081204/Ohana/src/opihi/cmd.data/write_vectors.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/cmd.data/write_vectors.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/cmd.data/write_vectors.c	(revision 20890)
@@ -45,4 +45,5 @@
       gprint (GP_ERR, "USAGE: write (file) vector vector ...\n");
       fclose (f);
+  fflush (f);
       return (FALSE);
   }
@@ -56,4 +57,5 @@
       free (vec);
       fclose (f);
+  fflush (f);
       return (FALSE);    
     }
@@ -67,4 +69,5 @@
       free (vec);
       fclose (f);
+  fflush (f);
       return (FALSE);    
     }
@@ -81,4 +84,5 @@
     fclose (f);
     free (vec);
+  fflush (f);
     return (TRUE);
   }
@@ -103,4 +107,5 @@
       free (format);
       fclose (f);
+  fflush (f);
       return (FALSE);
     }
@@ -139,4 +144,5 @@
   }
   fclose (f);
+  fflush (f);
 
   free (fmttype);
Index: branches/bills_081204/Ohana/src/opihi/dimm/camera.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dimm/camera.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dimm/camera.c	(revision 20890)
@@ -112,5 +112,5 @@
     ReadOut (x, y, dx, dy, 1, buf[0].matrix.buffer);
 
-    gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, gfits_get_unsign_mode());
+    gfits_convert_format (&buf[0].header, &buf[0].matrix, -32, 1.0, 0.0, 0xffff, gfits_get_unsign_mode());
 
     EXIT_STATUS (TRUE);
Index: branches/bills_081204/Ohana/src/opihi/doc/opihi-vectors.txt
===================================================================
--- branches/bills_081204/Ohana/src/opihi/doc/opihi-vectors.txt	(revision 20890)
+++ branches/bills_081204/Ohana/src/opihi/doc/opihi-vectors.txt	(revision 20890)
@@ -0,0 +1,21 @@
+
+2008.04.20
+
+  I am trying to update the opihi vector code for two important upgrades:
+
+  1) I want to convert the float vector to double vectors
+  2) I want to allow the option of int (unsigned int) vectors, especially for flags
+
+  I've define opihi_float and opihi_int types.  A major portion of the
+  upgrade is to convert all of the places where a vector is allocated
+  or manipulated as a float to use 'opihi_float' which can then be
+  re-defined.  The other trick here is to identify the functions which
+  require a vector to be a float and set assserts or error reporting.
+
+  The other major change is to modify the dvo math code.  for the
+  moment, all functions should result in opihi_float vectors except
+  for the explicit cases of the db extractions which return flags.
+
+  Some math operations on the vectors will force conversion to floats,
+  but some are sensible as functions from int to int, and those
+  results can be kept in their native format.
Index: branches/bills_081204/Ohana/src/opihi/dvo/avextract.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/avextract.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/avextract.c	(revision 20890)
@@ -55,5 +55,5 @@
 
   // parse the remainder of the line as a boolean math expression
-  cstack = isolate_elements (argc-last, &argv[last], &Ncstack);
+  cstack = isolate_elements (argc-next, &argv[next], &Ncstack);
   
   // construct the db Boolean math stack (frees cstack)
@@ -65,5 +65,5 @@
   // parse stack elements into fields and scalars as needed
   Nreturn = Nfields; 
-  dbCheckStack (stack, Nstack, DVO_TABLE_AVERAGE, &fields, &Nfields);
+  if (!dbCheckStack (stack, Nstack, DVO_TABLE_AVERAGE, &fields, &Nfields)) goto escape;
   // XXX handle errors
 
Index: branches/bills_081204/Ohana/src/opihi/dvo/ccd.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/ccd.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/ccd.c	(revision 20890)
@@ -7,4 +7,5 @@
   int N1, N2, i1, i2, mode[4];
   int Nsecfilt, KeepNulls;
+  void *Signal;
 
   Catalog catalog;
@@ -55,6 +56,10 @@
   if ((yvec = SelectVector ("yv", ANYVECTOR, TRUE)) == NULL) goto escape;
 
+  // grab data from all selected sky regions
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
   /* loop over regions, extract data for each region */
-  for (k = 0; k < skylist[0].Nregions; k++) {
+  for (k = 0; (k < skylist[0].Nregions) && !interrupt; k++) {
     /* lock, load, unlock catalog */
     catalog.filename = skylist[0].filename[k];
@@ -70,5 +75,5 @@
 
     /* get correct mags, convert to X,Y */
-    for (i = 0; i < catalog.Naverage; i++) {
+    for (i = 0; (i < catalog.Naverage) && !interrupt; i++) {
       M1 = M2 = NULL;
       m = catalog.average[i].measureOffset;
Index: branches/bills_081204/Ohana/src/opihi/dvo/cmd.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/cmd.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/cmd.c	(revision 20890)
@@ -7,4 +7,5 @@
   int Npts, NPTS, mode[3];
   int Nsecfilt, KeepNulls;
+  void *Signal;
 
   PhotCode *code[3];
@@ -53,6 +54,10 @@
   if ((yvec = SelectVector ("yv", ANYVECTOR, TRUE)) == NULL) goto escape;
 
+  // grab data from all selected sky regions
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
   /* loop over regions, extract data for each region */
-  for (j = 0; j < skylist[0].Nregions; j++) {
+  for (j = 0; (j < skylist[0].Nregions) && !interrupt; j++) {
     /* lock, load, unlock catalog */
     catalog.filename = skylist[0].filename[j];
@@ -68,5 +73,5 @@
     
     /* get correct mags, convert to X,Y */
-    for (i = 0; i < catalog.Naverage; i++) {
+    for (i = 0; (i < catalog.Naverage) && !interrupt; i++) {
       M1 = M3 = NULL;
       m = catalog.average[i].measureOffset;
Index: branches/bills_081204/Ohana/src/opihi/dvo/dbCheckStack.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/dbCheckStack.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/dbCheckStack.c	(revision 20890)
@@ -23,4 +23,37 @@
       } 
 
+      // the value might be a special type of number: RA,DEC in sexigesimal or 
+      // time in YYYY/MM/DD, etc
+      // status = ohana_str_to_time (stack[i].name, &seconds);
+      // keep as time_t
+      // status = ohana_dms_to_ddd (&ddd, stack[i].name);
+      // keep as either hours (RA) or deg (DEC).
+
+      // strings protected by double quotes should parse to be special types of numbers
+      // (photcodes, dates, ra / dec)
+      if (stack[i].name[0] == '"') {
+	  if (stack[i].name[strlen(stack[i].name) - 1] != '"') {
+	      gprint (GP_ERR, "syntax error for field %s\n", stack[i].name);
+	      goto failure;
+	  }
+	  char *tmpstring;
+	  tmpstring = strncreate (&stack[i].name[1], strlen(stack[i].name) - 2);
+
+	  // attempt to parse the string as a special word:
+	  PhotCode *code;
+	  code = GetPhotcodebyName (tmpstring);
+	  if (code) {
+	      stack[i].Float = code->code;
+	      stack[i].type  = 'S';
+	      continue;
+	  } 
+
+	  // add in tests for sexigesimal, date/time
+	  // TimeRefPM = ohana_date_to_sec ("2000/01/01");
+	  
+	  gprint (GP_ERR, "syntax error for field %s\n", stack[i].name);
+	  goto failure;
+      }
+
       // this must be a field : is it already in the list?
       for (j = 0; (j < Nfields) && strcasecmp (stack[i].name, fields[j].name); j++);
@@ -42,5 +75,5 @@
       if (!status) {
 	gprint (GP_ERR, "unknown database field %s\n", stack[i].name);
-	return (FALSE);
+	goto failure;
       }
       stack[i].field = Nfields;
@@ -56,6 +89,10 @@
   *inNfields = Nfields;
   *inFields = fields;
+  return (TRUE);
 
-  return (TRUE);
+failure:
+  *inNfields = Nfields;
+  *inFields = fields;
+  return (FALSE);
 }
 
Index: branches/bills_081204/Ohana/src/opihi/dvo/gcat.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/gcat.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/gcat.c	(revision 20890)
@@ -8,4 +8,5 @@
   SkyTable *sky;
   SkyList *skylist;
+  void *Signal;
 
   if ((argc != 3) && (argc != 4)) {
@@ -25,5 +26,9 @@
   skylist = SkyListByRadius (sky, -1, Ra, Dec, Radius);
 
-  for (i = 0; i < skylist[0].Nregions; i++) {
+  // prepare to handle interrupt signals
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
+  for (i = 0; (i < skylist[0].Nregions) && !interrupt; i++) {
     if (stat (skylist[0].filename[i], &filestat) != -1) {
       gprint (GP_ERR, "%3d %s *\n", i, skylist[0].regions[i][0].name);
Index: branches/bills_081204/Ohana/src/opihi/dvo/imdata.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/imdata.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/imdata.c	(revision 20890)
@@ -15,4 +15,5 @@
   Vector *vec;
   SkyRegionSelection *selection;
+  void *Signal;
 
   // parse skyregion options
@@ -127,6 +128,10 @@
   vec[0].Nelements = N = 0;
 
+  // prepare to handle interrupt signals
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
   /* for each region file, extract the data of interest in the right time range */
-  for (j = 0; j < skylist[0].Nregions; j++) {
+  for (j = 0; (j < skylist[0].Nregions) && !interrupt; j++) {
 
     /* get file name and open */
Index: branches/bills_081204/Ohana/src/opihi/dvo/mextract.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/mextract.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/mextract.c	(revision 20890)
@@ -70,5 +70,5 @@
   // parse stack elements into fields and scalars as needed
   Nreturn = Nfields; 
-  dbCheckStack (stack, Nstack, DVO_TABLE_MEASURE, &fields, &Nfields);
+  if (!dbCheckStack (stack, Nstack, DVO_TABLE_MEASURE, &fields, &Nfields)) goto escape;
   // XXX handle errors
 
@@ -105,4 +105,5 @@
   Signal = signal (SIGINT, handle_interrupt);
   interrupt = FALSE;
+
   for (i = 0; (i < skylist[0].Nregions) && !interrupt; i++) {
     /* lock, load, unlock catalog */
Index: branches/bills_081204/Ohana/src/opihi/dvo/mmextract.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/mmextract.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/mmextract.c	(revision 20890)
@@ -120,6 +120,6 @@
   Nreturn = 2*Nfields; // we are returning fieldi_1, fieldi_2 for the selected fields
 
-  dbCheckStack (stack1, Nstack1, DVO_TABLE_MEASURE, &fields, &Nfields);
-  dbCheckStack (stack2, Nstack2, DVO_TABLE_MEASURE, &fields, &Nfields);
+  if (!dbCheckStack (stack1, Nstack1, DVO_TABLE_MEASURE, &fields, &Nfields)) goto escape;
+  if (!dbCheckStack (stack2, Nstack2, DVO_TABLE_MEASURE, &fields, &Nfields)) goto escape;
   // XXX handle errors
 
Index: branches/bills_081204/Ohana/src/opihi/dvo/paverage.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/paverage.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/paverage.c	(revision 20890)
@@ -6,9 +6,10 @@
   FILE *f;
   int i, j, kapa, Narg, Npts, NPTS, status, VERBOSE;
-  int Nsecfilt, Nsec;
+  int Nsecfilt, Nsec, Nloaded;
   double Mz, Mr, mag;
   double Radius, Rmin, Rmax, R, D;
   unsigned IDclip, IDchoice, LimExclude;
   float *Xvec, *Yvec, *Zvec;
+  void *Signal;
 
   PhotCode *photcode;
@@ -99,5 +100,10 @@
   ALLOCATE (Zvec, float, NPTS);
 
-  for (j = 0; j < skylist[0].Nregions; j++) {
+  // prepare to handle interrupt signals
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
+  Nloaded = 0;
+  for (j = 0; (j < skylist[0].Nregions) && !interrupt; j++) {
     catalog.filename = skylist[0].filename[j];
     catalog.catflags = LOAD_AVES | LOAD_SECF;
@@ -115,5 +121,5 @@
 
     /* project stars to screen display coords */
-    for (i = 0; i < catalog.Naverage; i++) {
+    for (i = 0; (i < catalog.Naverage) && !interrupt; i++) {
       if (IDclip && (average[i].code != IDchoice)) continue;
       average[i].R = ohana_normalize_angle (average[i].R);
@@ -128,8 +134,5 @@
       D = average[i].D;
       status = fRD_to_XY (&Xvec[Npts], &Yvec[Npts], R, D, &graphmode.coords);
-      if (!status) {
-	  fprintf (stderr, ".");
-	  continue;
-      }
+      if (!status) continue;
       Npts ++;
 
@@ -140,9 +143,11 @@
 	  REALLOCATE (Zvec, float, NPTS);
       }
-      if (Npts > NCHUNK) {
+      if ((Npts > NCHUNK) || (Nloaded >= 25)) {
 	  PlotVectorTriplet (kapa, Npts, Xvec, Yvec, Zvec, &graphmode);
 	  Npts = 0;
+	  Nloaded = 0;
       }
     }
+    Nloaded ++;
     dvo_catalog_free (&catalog);
   }
Index: branches/bills_081204/Ohana/src/opihi/dvo/pmeasure.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/pmeasure.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/pmeasure.c	(revision 20890)
@@ -7,5 +7,5 @@
   
   FILE *f;
-  int i, j, k, m, kapa, Narg, Npts, NPTS, status, VERBOSE, TimeSelect;
+  int i, j, k, m, kapa, Narg, Npts, NPTS, status, VERBOSE, TimeSelect, Nloaded;
   double Mz, Mr, mag;
   double Radius, Rmin, Rmax, R, D, trange;
@@ -16,4 +16,5 @@
   float *Xvec, *Yvec, *Zvec;
   time_t tzero, tend;
+  void *Signal;
 
   SkyTable *sky;
@@ -172,5 +173,10 @@
   ALLOCATE (Zvec, float, NPTS);
 
-  for (j = 0; j < skylist[0].Nregions; j++) {
+  // prepare to handle interrupt signals
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
+  Nloaded = 0;
+  for (j = 0; (j < skylist[0].Nregions) && !interrupt; j++) {
     catalog.filename = skylist[0].filename[j];
     catalog.catflags = LOAD_AVES | LOAD_MEAS;
@@ -185,5 +191,5 @@
 
     /* project stars to screen display coords */
-    for (i = 0; i < catalog.Naverage; i++) {
+    for (i = 0; (i < catalog.Naverage) && !interrupt; i++) {
       if (IDclip && (catalog.average[i].code != IDchoice)) continue;
       catalog.average[i].R = ohana_normalize_angle (catalog.average[i].R);
@@ -213,8 +219,5 @@
 	}
 	status = fRD_to_XY (&Xvec[Npts], &Yvec[Npts], R, D, &graphmode.coords);
-	if (!status) {
-	  fprintf (stderr, ".");
-	  continue;
-	}
+	if (!status) continue;
 	Npts ++;
 
@@ -225,10 +228,12 @@
 	    REALLOCATE (Zvec, float, NPTS);
 	}
-	if (Npts > NCHUNK) {
+	if ((Npts > NCHUNK) || (Nloaded >= 25)) {
 	    PlotVectorTriplet (kapa, Npts, Xvec, Yvec, Zvec, &graphmode);
 	    Npts = 0;
+	    Nloaded = 0;
 	}
       }
     }
+    Nloaded ++;
     dvo_catalog_free (&catalog);
   }
Index: branches/bills_081204/Ohana/src/opihi/dvo/skycat.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/dvo/skycat.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/dvo/skycat.c	(revision 20890)
@@ -14,4 +14,5 @@
   SkyList *skylist;
   SkyRegion **regions;
+  void *Signal;
 
   VERBOSE = FALSE;
@@ -60,5 +61,9 @@
   Nregions = skylist[0].Nregions;
 
-  for (i = 0; i < Nregions; i++) {
+  // prepare to handle interrupt signals
+  Signal = signal (SIGINT, handle_interrupt);
+  interrupt = FALSE;
+
+  for (i = 0; (i < Nregions) && !interrupt; i++) {
     if (ShowAll || (stat (skylist[0].filename[i], &filestat) != -1)) {
       if (VERBOSE) gprint (GP_ERR, "%3d %s %6.2f - %6.2f, %6.2f - %6.2f\n", i, regions[i][0].name, 
Index: branches/bills_081204/Ohana/src/opihi/lib.shell/multicommand.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/opihi/lib.shell/multicommand.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/opihi/lib.shell/multicommand.c	(revision 20890)
@@ -88,5 +88,5 @@
 		exit (32);
 	      default:
-		gprint (GP_ERR, "server is busy...\n");
+		gprint (GP_ERR, "server is busy...32\n");
 		bufferPending = TRUE;
 		goto escape;
@@ -101,5 +101,5 @@
 		exit (33);
 	      default:
-		gprint (GP_ERR, "server is busy...\n");
+		gprint (GP_ERR, "server is busy...33\n");
 		bufferPending = TRUE;
 		goto escape;
@@ -116,5 +116,5 @@
 		exit (34);
 	      default:
-		gprint (GP_ERR, "server is busy...\n");
+		gprint (GP_ERR, "server is busy...34\n");
 		bufferPending = TRUE;
 		goto escape;
@@ -131,5 +131,5 @@
 		exit (35);
 	      default:
-		gprint (GP_ERR, "server is busy...\n");
+		gprint (GP_ERR, "server is busy...35\n");
 		bufferPending = TRUE;
 		goto escape;
Index: branches/bills_081204/Ohana/src/relphot/src/GridOps.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/relphot/src/GridOps.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/relphot/src/GridOps.c	(revision 20890)
@@ -19,7 +19,7 @@
 static int    **Ymeas;
 
-static int    **clist;
-static int    **mlist;
-static int     *Nlist;
+static int    **clist; // link from measurement on a cell to catalog containing measurement
+static int    **mlist; // link from measurement on a cell to measurement in a catalog
+static int     *Nlist; // list of measurements for each grid cell
 static int     *NLIST;
 
@@ -282,4 +282,239 @@
 }
 
+/* direct (non-iterative) solution for Mgrid values for all grid bins */
+void setMgridDirect (Catalog *catalog, int Ncatalog) {
+
+  int **gotstar, **gridmeas;
+  int i, j, k, N, Nmax, Ngood, Nbad, Nmos, Ncal, Nrel, Nsys, Nfit;
+  double **A, **B, *Mjx, *Wjx;
+  float Msys, Mrel, Mcal, Mmos, Merr, Wsys;
+  double Mj, Wj;
+  
+  if (!USE_GRID) return;
+
+  ALLOCATE (A, double *, Ngrid);
+  ALLOCATE (B, double *, Ngrid);
+  for (i = 0; i < Ngrid; i++) {
+    ALLOCATE (A[i], double, Ngrid);
+    ALLOCATE (B[i], double, 1);
+    memset (A[i], 0, Ngrid*sizeof(double));
+    memset (B[i], 0, sizeof(double));
+  }
+
+  Ngood = Nbad = Ncal = Nmos = Nrel = Nsys = 0;
+
+  ALLOCATE (gotstar, int *, Ncatalog);
+  for (i = 0; i < Ncatalog; i++) {
+    ALLOCATE (gotstar[i], int, catalog[i].Naverage);
+  }  
+
+  // set up gridmeas table : grid index for each measurement
+  ALLOCATE (gridmeas, int *, Ncatalog);
+  for (i = 0; i < Ncatalog; i++) {
+    ALLOCATE (gridmeas[i], int, catalog[i].Nmeasure);
+    for (j = 0; j < catalog[i].Nmeasure; j++) {
+      gridmeas[i][j] = -1;
+    }
+  }  
+  for (i = 0; i < Ngrid; i++) {
+    for (j = 0; j < Nlist[i]; j++) {
+      int m, c;
+      m = mlist[i][j];
+      c = clist[i][j];
+      gridmeas[c][m] = i;
+    }
+  }
+
+  // as we loop over the grid cells, we need to accumulate the values of Wjx for each star
+  ALLOCATE (Wjx, double, Ngrid);
+  ALLOCATE (Mjx, double, Ngrid);
+
+  // accumulate the elements of the matrix equation.  We have an equation of the form: Ax = B
+  // where x is the vector of grid cell values G_x (x = 0 - Ngrid), A is an Ngrid x Ngrid matrix,
+  // and B is an Ngrid vector.  For a cell A(x,y), we need the following elements from each
+  // star which touches grid cell (x):
+  // 
+  // Mj  : sum over all measurements of Msys / dMsys^2 
+  // Wj  : sum over all measurements of 1.0 / dMsys^2 
+  // Mjx : sum over all measurements which touch cell (x) of Msys / dMsys^2
+  // Wjx : sum over all measurements which touch cell (x) of 1.0 / dMsys^2
+
+  // Mjx and Wjx can be calculated by summing over all measurements which touch the cell 
+  // Mj requires looping over stars which touch (x)
+
+  // this is tricky because we need to know both the measurements which touch a cell
+  // and the stars for which any measurement touches a cell.  we need to accumulate
+  // sums for each star which touches as cell on both bases.
+
+  for (i = 0; i < Ngrid; i++) {
+    
+    for (j = 0; j < Ncatalog; j++) {
+      memset (gotstar[j], 0, catalog[j].Naverage*sizeof(int));
+    }  
+
+    // we are looping over the stars, but doing so by looping over the set of measurements:
+    // every star which touches this grid cell has a measurement in Nlist[i]
+    for (j = 0; j < Nlist[i]; j++) {
+      
+      int mx, c, n, m0, Npts;
+
+      mx = mlist[i][j];
+      c  = clist[i][j];
+      n  = catalog[c].measure[mx].averef;
+      
+      // if we have already visited this star, skip the stuff below
+      if (gotstar[c][n]) continue;
+      gotstar[c][n] = TRUE;
+
+      // skip stars marked as BAD
+      if (catalog[c].average[n].code & STAR_BAD) {
+	Nrel ++;
+	continue;
+      }
+
+      m0 = catalog[c].average[n].measureOffset;
+
+      // we accumuate an entry for each cell
+      memset (Wjx, 0, Ngrid*sizeof(double));
+      memset (Mjx, 0, Ngrid*sizeof(double));
+      Npts = Mj = Wj = 0.0;
+
+      // if we have not yet visited this star, accumulate the Mj, Wj entries
+      for (k = 0; k < catalog[c].average[n].Nmeasure; k++) {
+
+	int m, Ng;
+
+	m = m0 + k;
+
+	// skip measurements marked as BAD
+	if (catalog[c].measure[m].dbFlags & MEAS_BAD) {
+	  Nbad ++;
+	  continue;
+	}
+
+	// skip images marked as BAD
+	Mcal = getMcal  (m, c);
+	if (isnan(Mcal)) {
+	  Ncal ++;
+	  continue;
+	}
+
+	// skip mosaics marked as BAD
+	Mmos = getMmos  (m, c);
+	if (isnan(Mmos)) {
+	  Nmos ++;
+	  continue;
+	}
+
+	// select the color- and airmass-corrected observed magnitude for this star
+	// XXX need to be able to turn off the color-correction until initial average mags are found
+	// Msys = PhotSys (&catalog[c].measure[m], &catalog[c].average[n], &catalog[c].secfilt[n*PhotNsec]);
+	Msys = PhotCat (&catalog[c].measure[m]);
+	if (isnan(Msys)) {
+	  Nsys++;
+	  continue;
+	}
+
+	// mag-error for this measurement
+	Merr =  MAX (catalog[c].measure[m].dM, MIN_ERROR);
+
+	// Wsys = 1.0 / SQ(Merr);
+	Wsys = 1.0;
+
+	Ng = gridmeas[c][m];
+	if (Ng == -1) continue;  // skip measurements which do not touch any cell
+
+	Mj += Msys * Wsys;  // we are only including measurements touching this cell
+	Wj += Wsys;  // we are only including measurements touching this cell
+	Npts ++;
+	Ngood ++;
+
+	Mjx[Ng] += Msys * Wsys;  // we are only including measurements touching cell (x)
+	Wjx[Ng] += Wsys;  // we are only including measurements touching cell (x)
+      }
+
+      // some stars will not have any valid measurements, skip these
+      if (Npts == 0) continue;
+
+      B[i][0] += Mj*Wjx[i]/Wj - Mjx[i];
+      A[i][i] -= Wjx[i];
+      for (k = 0; k < Ngrid; k++) {
+	A[i][k] += Wjx[i]*Wjx[k]/Wj;
+	// fprintf (stderr, "%3.0f ", Wjx[k]);
+      }
+    }
+  }
+
+  if (1) {
+
+    FILE *f;
+    Header header, theader;
+    Matrix matrix;
+
+    /* we are writing to this file */
+    f = fopen ("matrix.fits", "w");
+    if (f == (FILE *) NULL) { 
+      fprintf (stderr, "cannot open matrix.fits for output\n");
+      return;
+    }
+
+    /* save grid mag values */
+    gfits_init_header (&theader);
+    theader.Naxes = 2;
+    theader.Naxis[0] = Ngrid;
+    theader.Naxis[1] = Ngrid;
+    theader.bitpix   = -32;
+    gfits_create_Theader (&theader, "IMAGE");
+    gfits_modify (&theader, "EXTNAME", "%s", 1, "MATRIX");
+    gfits_create_matrix  (&theader, &matrix);
+    for (i = 0; i < Ngrid; i++) {
+      for (j = 0; j < Ngrid; j++) {
+	gfits_set_matrix_value (&matrix, i, j, (double) A[i][j]);
+      }
+    }
+    gfits_fwrite_header (f, &theader);
+    gfits_fwrite_matrix (f, &matrix);
+    gfits_free_matrix (&matrix);
+
+    gfits_modify (&theader, "EXTNAME", "%s", 1, "TRPOSE");
+    gfits_create_matrix  (&theader, &matrix);
+    for (i = 0; i < Ngrid; i++) {
+      for (j = 0; j < Ngrid; j++) {
+	gfits_set_matrix_value (&matrix, i, j, (double) A[j][i]);
+      }
+    }
+    gfits_fwrite_header (f, &theader);
+    gfits_fwrite_matrix (f, &matrix);
+    gfits_free_matrix (&matrix);
+    fclose (f);
+  }
+
+  dgaussjordan (A, B, Ngrid, 1);
+
+  fprintf (stderr, "grid cells fitted (Ngood: %d, Nbad: %d, Nmos: %d, Ncal: %d, Nrel: %d, Nsys: %d)\n", Ngood, Nbad, Nmos, Ncal, Nrel, Nsys);
+
+  for (i = 0; i < Ngrid; i++) {
+    gridM[i] = B[i][0];
+    gridS[i] = sqrt(A[i][i]);
+    gridN[i] = N;
+  }
+
+  free (Wjx);
+  free (Mjx);
+
+  for (i = 0; i < Ngrid; i++) {
+    free (A[i]);
+  }
+  free (A);
+  free (B);
+
+  for (i = 0; i < Ncatalog; i++) {
+    free (gotstar[i]);
+    free (gridmeas[i]);
+  }  
+  free (gotstar);
+  free (gridmeas);
+}
+
 /* determine Mgrid values for all grid bins */
 void setMgrid (Catalog *catalog) {
@@ -437,5 +672,5 @@
 void dump_grid () { 
 
-    int i, j, Nimage, Nbytes, Nformat;
+  int i, j, Nimage, Nbytes, Nformat;
   int *imlist;
   FILE *f;
Index: branches/bills_081204/Ohana/src/relphot/src/relphot.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/relphot/src/relphot.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/relphot/src/relphot.c	(revision 20890)
@@ -1,3 +1,4 @@
 # include "relphot.h"
+# define USE_DIRECT 0
 
 int main (int argc, char **argv) {
@@ -74,5 +75,21 @@
       int star_toofew;
 
+# if (USE_DIRECT)
+      // until we finish the grid analysis, do not reject stars out-of-hand based on ID_STAR_FEW
+      // XXX this is kind of poor: need to have a better distinctions about STAR_BAD in setMrel vs getMrel
+      star_toofew = STAR_TOOFEW;
+      STAR_TOOFEW = 0;
+      STAR_BAD  = ID_STAR_POOR;
+
       showGridCount ();
+      setMgridDirect (catalog, Ncatalog);
+
+      STAR_BAD  = ID_STAR_POOR | ID_STAR_FEW;
+      STAR_TOOFEW = star_toofew;
+
+      dump_grid ();
+      exit (0);
+
+# else
 
       // until we finish the grid analysis, do not reject stars out-of-hand based on ID_STAR_FEW
@@ -88,4 +105,5 @@
       STAR_BAD  = ID_STAR_POOR | ID_STAR_FEW;
       STAR_TOOFEW = star_toofew;
+# endif
   }
 
Index: branches/bills_081204/Ohana/src/tools/src/medianfilter.c
===================================================================
--- branches/cnb_branch_20081104/Ohana/src/tools/src/medianfilter.c	(revision 20530)
+++ branches/bills_081204/Ohana/src/tools/src/medianfilter.c	(revision 20890)
@@ -51,5 +51,5 @@
   gfits_read_header (filename[0], &head);
   gfits_create_matrix (&head, &out);
-  gfits_convert_format (&head, &out, -32, 1.0, 0.0, FALSE);
+  gfits_convert_format (&head, &out, -32, 1.0, 0.0, 0xffff, FALSE);
 
   /* find size of temporary images */
@@ -90,5 +90,5 @@
       tmphead[i].Naxis[0] = 1;
       tmphead[i].Naxis[1] = Npixrd;
-      gfits_convert_format (&tmphead[i], &tmpmatr[i], -32, 1.0, 0.0, FALSE);
+      gfits_convert_format (&tmphead[i], &tmpmatr[i], -32, 1.0, 0.0, 0xffff, FALSE);
     }
     
Index: branches/bills_081204/dbconfig/changes.txt
===================================================================
--- branches/cnb_branch_20081104/dbconfig/changes.txt	(revision 20530)
+++ branches/bills_081204/dbconfig/changes.txt	(revision 20890)
@@ -683,2 +683,36 @@
 ALTER TABLE flatcorrChipLink ADD COLUMN include tinyint after chip_id;
 ALTER TABLE flatcorrCamLink ADD COLUMN include tinyint after cam_id;
+
+ALTER TABLE rawImfile ADD COLUMN ignored TINYINT DEFAULT 0 AFTER moon_phase;
+
+-- add new tables to control magic de-streaking
+CREATE TABLE magicDSRun (
+        magic_ds_id BIGINT AUTO_INCREMENT,
+        magic_id BIGINT,
+        state VARCHAR(64),
+        stage VARCHAR(64),
+        outroot VARCHAR(255),
+        recoveryroot VARCHAR(255),
+        re_place TINYINT,
+        remove TINYINT,
+        PRIMARY KEY(magic_ds_id),
+        KEY(magic_ds_id),
+        KEY(state),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicDSFile (
+    magic_ds_id BIGINT,
+    component VARCHAR(64),
+    backup_path_base VARCHAR(255),
+    recovery_path_base VARCHAR(255),
+    fault SMALLINT,
+    PRIMARY KEY(magic_ds_id, component),
+    KEY(fault),
+    FOREIGN KEY (magic_ds_id) REFERENCES magicDSRun(magic_ds_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- add to the run table to make the queries far less expensive
+alter table magicDSRun add column stage_id BIGINT after stage;
+alter table magicDSRun add column cam_id BIGINT after stage_id;
+
Index: branches/bills_081204/dbconfig/magic.md
===================================================================
--- branches/cnb_branch_20081104/dbconfig/magic.md	(revision 20530)
+++ branches/bills_081204/dbconfig/magic.md	(revision 20890)
@@ -1,3 +1,3 @@
-# $Id: magic.md,v 1.10 2008-07-23 01:52:30 price Exp $
+# $Id: magic.md,v 1.12 2008-11-26 03:21:13 bills Exp $
 
 ### Fault in magicRun indicates that the processing tree failed
@@ -40,8 +40,22 @@
 END
 
-### I don't think we need this if magic produces mask descriptions. -- PAP.
-#magicSkyfileMask METADATA
-#    magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
-#    diff_id     S64         0       # Primary Key fkey(magic_id, diff_id) ref magicInputSkyfile(magic_id, diff_id)
-#    uri         STR         255     #  fkey(magic_id) ref magicMask(magic_id)
-#END
+magicDSRun METADATA
+    magic_ds_id S64         0       # Primary Key fkey(magic_ds_id)
+    magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
+    state       STR         0       # Key
+    stage       STR         64
+    stage_id    S64         0
+    cam_id      S64         0
+    outroot     STR         255
+    recoveryroot    STR     255
+    re_place    BOOL        f
+    remove      BOOL        f
+END
+
+magicDSFile METADATA
+    magic_ds_id S64         0       # Primary Key
+    component   STR         64
+    backup_path_base  STR         255
+    recovery_path_base STR        255
+    fault       S16         0
+END
Index: branches/bills_081204/dbconfig/tasks.md
===================================================================
--- branches/cnb_branch_20081104/dbconfig/tasks.md	(revision 20530)
+++ branches/bills_081204/dbconfig/tasks.md	(revision 20890)
@@ -1,3 +1,3 @@
-# $Id: tasks.md,v 1.157 2008-10-22 02:39:15 bills Exp $
+# $Id: tasks.md,v 1.158 2008-11-09 23:51:01 price Exp $
 
 # this table records all exposure ID ever seen from the summit
@@ -235,4 +235,5 @@
     moon_alt    F32         0.0
     moon_phase  F32         0.0
+    ignored	BOOL        FALSE
     hostname    STR         64
     fault       S16         0       # Key NOT NULL
Index: branches/bills_081204/ippScripts/Build.PL
===================================================================
--- branches/cnb_branch_20081104/ippScripts/Build.PL	(revision 20530)
+++ branches/bills_081204/ippScripts/Build.PL	(revision 20890)
@@ -57,4 +57,5 @@
         scripts/magic_process.pl
         scripts/magic_mask.pl
+        scripts/magic_destreak.pl
         scripts/ipp_cleanup.pl
         scripts/ipp_inject_fileset.pl
Index: branches/bills_081204/ippScripts/scripts/camera_exp.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/camera_exp.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/camera_exp.pl	(revision 20890)
@@ -60,9 +60,16 @@
 
 my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
+
+if (not defined $run_state) { $run_state = 'new'; }
 if ($run_state eq 'update') {
     $logDest .= '.update';
 }
 
-$ipprc->redirect_output($logDest) if $redirect;
+if ($redirect) {
+    $ipprc->redirect_output($logDest);
+    print "\n\n";
+    print "Starting script $0 on $host\n\n";
+    print "COMMAND IS: @ARGV\n\n";
+}
 
 # Recipes to use based on reduction class
@@ -145,4 +152,7 @@
 my ($list3File, $list3Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => !$save_temps ); # For astrometry
 
+### XXX for the moment, always generate the bright-star mask
+my ($list4File, $list4Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b4.list.XXXX", UNLINK => !$save_temps ); # For astrometry
+
 # XXX we perform astrometry iff photometry output exists
 my $chipObjectsExist = 0;
@@ -154,4 +164,5 @@
     # we expect the chip analysis stage to produce psphot output (cmf file) and two binned images
     my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id);
+    my $chipMask   = $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
 
     # if any of the output chip photometry files exist, we can run psastro / addstar below
@@ -166,8 +177,10 @@
     print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n");
     print $list3File ($ipprc->file_resolve($chipObjects, 0) . "\n");
+    print $list4File ($ipprc->file_resolve($chipMask, 0) . "\n");
 }
 close $list1File;
 close $list2File;
 close $list3File;
+close $list4File;
 
 # Output products
@@ -234,5 +247,5 @@
         # run psastro on the chipObjects, producing fpaObjects
         my $command;
-        $command  = "$psastro -list $list3Name $outroot";
+        $command  = "$psastro -list $list3Name -masklist $list4Name $outroot";
         $command .= " -tracedest $traceDest -log $logDest";
         $command .= " -dbname $dbname" if defined $dbname;
Index: branches/bills_081204/ippScripts/scripts/ds9_cmf_regions.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/ds9_cmf_regions.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/ds9_cmf_regions.pl	(revision 20890)
@@ -69,7 +69,16 @@
 
 my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => !$save_temps );
+my $numGood = 0;                # Number of good sources
+my $numBad = 0;                 # Number of bad sources
 for (my $i = 0; $i < $numRows; $i++) {
-    print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$radius) \# color = " .
-        (($$flags[$i] & $flag) ? $flag_colour : $colour) . "\n";
+    my $col;                    # Colour to use
+    if ($$flags[$i] & $flag) {
+        $numBad++;
+        $col = $flag_colour;
+    } else {
+        $numGood++;
+        $col = $colour;
+    }
+    print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$radius) \# color = $col\n";
 }
 close $coordFile;
@@ -85,5 +94,5 @@
 xpaset(@settings);
 
-print "Plotted $numRows sources.\n";
+print "Plotted $numRows ($numGood good, $numBad bad) sources.\n";
 
 ### Pau.
Index: branches/bills_081204/ippScripts/scripts/flatcorr_proc.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/flatcorr_proc.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/flatcorr_proc.pl	(revision 20890)
@@ -152,5 +152,5 @@
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+        run(command => $command, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -178,5 +178,5 @@
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+        run(command => $command, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
Index: branches/bills_081204/ippScripts/scripts/magic_definerun.pl
===================================================================
--- branches/bills_081204/ippScripts/scripts/magic_definerun.pl	(revision 20890)
+++ branches/bills_081204/ippScripts/scripts/magic_definerun.pl	(revision 20890)
@@ -0,0 +1,232 @@
+#!/usr/bin/env perl
+
+# program to define a magic run
+# The magictool mode will not queue a run until all diffs from an exposure are
+# successfully completed. There may also be issues with multiple warp runs
+# for a single exposure.
+# This program selects the diffs based on the value for warp_id if provided
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Parse the command-line arguments
+my ($exp_id, $warp_id, $min_diff_id, $label, $workdir, $dbname, $save_temps, $verbose);
+
+GetOptions(
+           'exp_id=s'        => \$exp_id,     # exposure identifier
+           'warp_id=s'       => \$warp_id,    # warp identifier
+           'min_diff_id=s'   => \$min_diff_id, # minimum diff id to consider
+           'label=s'         => \$label,      # label for images of interest
+           'dbname=s'        => \$dbname,     # Database name
+           'workdir=s'       => \$workdir,    # workdir
+           'save-temps'      => \$save_temps, # Save temporary files?
+           'verbose'         => \$verbose,    # Print stuff?
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --warp_id --workdir",
+           -exitval => 3) unless
+    defined $workdir and
+    defined $warp_id;
+
+# $ipprc->define_camera($camera);
+
+# Look for programs we need
+my $missing_tools;
+my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool  = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $warptool  = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+if (!$exp_id) {
+    my $command = "$warptool -warped -warp_id $warp_id -limit 1";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform warptool -warped error_code: $error_code", $error_code);
+    }
+
+    my $warptool_output = join "", @$stdout_buf;
+
+    my $warpskyfiles = parse_md_fast($mdcParser, $warptool_output);
+    &my_die("Unable to parse metadata list", $PS_EXIT_UNKNOWN_ERROR) unless $warpskyfiles;
+
+    my $warped = $warpskyfiles->[0];
+    $exp_id = $warped->{exp_id};
+    &my_die("failed to get exp_id for warp: $warp_id", $PS_EXIT_UNKNOWN_ERROR) unless $exp_id;
+}
+
+### Get a list of inputs
+my $inputs;                     # List of inputs
+{
+    my $command = "$difftool -diffskyfile -exp_id $exp_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform difftool -diffskyfile error_code: $error_code", $error_code);
+    }
+
+    my $difftool_output = join "", @$stdout_buf;
+
+    $inputs = parse_md_fast($mdcParser, $difftool_output);
+    &my_die("Unable to parse metadata list", $PS_EXIT_UNKNOWN_ERROR) unless $inputs;
+}
+
+my $num_skyfiles = @$inputs;
+if (! $num_skyfiles) {
+    carp("failed to find any diffskyfiles for exposure $exp_id");
+    exit(1);
+}
+
+print "$num_skyfiles skyfiles found for $exp_id\n" if $verbose;
+
+if ($warp_id or $min_diff_id) {
+    # filter the inputs
+    my $i = 0;
+    while ($i < scalar @$inputs) {
+        my $keep = 1;
+        my $sf = $inputs->[$i];
+
+        if ($warp_id) {
+            if ($warp_id ne $sf->{warp_id_temp_0}) {;
+                $keep = 0;
+            }
+        }
+        if ($min_diff_id) {
+            if ($sf->{diff_id} < $min_diff_id) {
+                $keep = 0;
+            }
+        }
+        if ( $keep) {
+            $i++;
+        } else {
+            # remove it
+            splice @$inputs, $i, 1;
+        }
+    }
+    $num_skyfiles = $i;
+    print "$num_skyfiles skyfiles accepted for $exp_id\n" if $verbose;
+}
+
+if (!$num_skyfiles) {
+    # should I return an error here?
+    &my_die("no skyfiles matching criteria found for exp_id $exp_id", 0);
+}
+
+my $magic_id;
+{
+    my $command = "$magictool -definerun -exp_id $exp_id -workdir $workdir -simple";
+    $command .= " -label $label" if defined $label;
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    print "$command\n";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform difftool -diffskyfile: $error_code", $error_code);
+    }
+
+    my $buffer = join "", @$stdout_buf;
+    ($magic_id, undef) = split " ", $buffer;
+}
+
+&my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id;
+
+foreach my $diff_skyfile (@$inputs) {
+    my $skycell_id = $diff_skyfile->{skycell_id};
+    if ($diff_skyfile->{fault} > 0) {
+        print "skipping faulted diffSkyfile: $skycell_id\n" if $verbose;
+        next;
+    }
+    my $diff_id = $diff_skyfile->{diff_id};
+    my $command = "$magictool -addinputskyfile -magic_id $magic_id -diff_id $diff_id";
+    $command .= "  -node $skycell_id";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform magictool -addinputskyfile for magicRun: $magic_id skycell_id: $skycell_id: $error_code", $error_code);
+    }
+}
+
+# set the new magicRun to state 'run'
+{
+    my $command = "$magictool -updaterun -magic_id $magic_id -state run";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform magictool -updaterun for magicRun: $magic_id: $error_code", $error_code);
+    }
+}
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $exit_code = shift;      # Exit code to add
+
+    carp($msg);
+    exit $exit_code;
+}
+
+sub parse_md_fast {
+    my $mdcParser = shift;
+    my $input = shift;
+    my $output = ();
+
+    my @whole = split /\n/, $input;
+    my @single = ();
+
+    my $n;
+    while ( ($n = @whole) > 0) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or
+                print STDERR "Unable to parse metdata config doc" and return undef;
+            push @$output, $list->[0];
+
+            @single = ();
+        }
+    }
+    return $output;
+}
+
+__END__
Index: branches/bills_081204/ippScripts/scripts/magic_destreak.pl
===================================================================
--- branches/bills_081204/ippScripts/scripts/magic_destreak.pl	(revision 20890)
+++ branches/bills_081204/ippScripts/scripts/magic_destreak.pl	(revision 20890)
@@ -0,0 +1,282 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Parse the command-line arguments
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($outroot, $recoveryroot);
+my ($replace, $remove);
+my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,     # camera for evaluating file rules
+           'streaks=s'      => \$streaks,    # file containing the list of streaks
+           'stage=s'        => \$stage,      # raw, chip, warp, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
+           'component=s'    => \$component,  # the class_id or skycell_id
+           'uri=s'          => \$uri,        # uri of the input image
+           'path_base=s'    => \$path_base,  # path_base of the input
+           'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
+           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'replace=s'        => \$replace,    # replace the input images with the results.
+           'remove=s'         => \$remove,     # remove the original images when done YIKES!
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --outroot",
+           -exitval => 3) unless
+    defined $magic_ds_id and
+    defined $camera and
+    defined $streaks and
+    defined $stage and
+    defined $stage_id and
+    defined $component and
+    defined $outroot;
+    
+
+my ($skycell_args, $class_id, $skycell_id);
+
+if (($stage eq "raw") or ($stage eq "chip")) {
+    $class_id = $component;
+    $skycell_args = " -class_id $component";
+} elsif ($stage eq "warp") {
+    $skycell_id = $component;
+    $skycell_args = " -skycell_id $component";
+} elsif ($stage eq "diff") {
+    $skycell_id = $component;
+} else {
+    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+$ipprc->define_camera($camera);
+
+
+# Look for programs we need
+my $missing_tools;
+my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
+my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
+# todo get rid of this dependence on magic_id
+my $magictool   = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $basename = basename($path_base);
+
+#
+# parse the replace and remove arguments check for errors and set upt
+# the appropriate paths
+if (defined($replace) and ($replace eq "F")) {
+    $replace = 0;
+}
+
+if (($stage eq "raw") and $replace and ! $recoveryroot) {
+    &my_die("Can not replace raw files without defining recoveryroot.",
+        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+if (defined($remove) and ($remove eq "F")) {
+    $remove = 0;
+}
+
+if ($remove and !$replace) {
+    &my_die("cannot remove without replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
+
+# create the output directories if it is not a nebulous path and it doesn't exist
+if (index($outroot, "neb://") != 0) {
+    if (! -e $outroot ) {
+        mkdir $outroot 
+            or &my_die("cannot create output directory $outroot", $magic_ds_id, $component, 
+                $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
+my $backup_path_base;
+if (! $remove) {
+    $backup_path_base = "$outroot/$basename";
+}
+
+my $recovery_path_base;
+if ($recoveryroot) {
+    if (index($recoveryroot, "neb://") != 0) {
+        if (! -e $recoveryroot ) {
+            mkdir $recoveryroot 
+                or &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
+                    $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+    $recovery_path_base = "$recoveryroot/$basename";
+}
+
+# get skycell list if needed
+my ($sfh, $skycell_list);          
+if ($skycell_args) {
+    my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+    }
+
+    my $getskycells_output = join "", @$stdout_buf;
+    if ($getskycells_output) {
+        my $metadata = $mdcParser->parse($getskycells_output) or
+            &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+        my $skycells = parse_md_list($metadata) or
+                &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+
+        foreach my $skycell (@$skycells) {
+            print $sfh "$skycell->{uri}\n"
+        }
+        close $sfh;
+    }
+}
+
+my $image = $uri;
+
+my ($mask, $weight, $astrom);
+
+if ($stage eq "raw") {
+    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+} elsif ($stage eq "chip") {
+    $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id);
+    $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $path_base, $class_id);
+    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+} elsif ($stage eq "warp") {
+    $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
+    $weight = $ipprc->filename("PSWARP.OUTPUT.WEIGHT", $path_base);
+} elsif ($stage eq "diff") {
+    $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
+    $weight = $ipprc->filename("PPSUB.OUTPUT.WEIGHT", $path_base);
+}
+
+{
+    my $command = "$streaksremove -stage $stage -tmproot $outroot -streaks $streaks -image $image";
+    $command .= " -class_id $class_id" if defined $class_id;
+    $command .= " -recovery $recoveryroot" if defined $recoveryroot;
+    $command .= " -astrom $astrom" if defined $astrom;
+    $command .= " -mask $mask" if defined $mask;
+    $command .= " -weight $weight" if defined $weight;
+    $command .= " -skycelllist $skycell_list" if defined $skycell_list;
+    $command .= " -dbname $dbname" if defined $dbname;
+    unless (defined $no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    }
+}
+
+# if recovery and/or backup files were expected make sure they exist
+
+# Input result into database
+{
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
+    $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, $component,
+                $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
+
+
+
+### Pau.
+
+
+sub file_check
+{
+    my $file = shift;           # Name of file
+    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
+        $ipprc->file_exists($file);
+}
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $magic_ds_id = shift;    # Magic DS identifier
+    my $component = shift;      # class_id or skycell_id
+    my $exit_code = shift;      # Exit code to add
+
+    my $command = "$magicdstool -adddestreakedfile";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -component $component";
+    $command   .= " -code $exit_code";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id $component");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__
Index: branches/bills_081204/ippScripts/scripts/magic_mask.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/magic_mask.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/magic_mask.pl	(revision 20890)
@@ -29,5 +29,5 @@
 
 # Parse the command-line arguments
-my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect);
+my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
 GetOptions(
            'magic_id=s'      => \$magic_id,   # Magic identifier
@@ -39,5 +39,5 @@
            'no-update'       => \$no_update,  # Don't update the database?
            'no-op'           => \$no_op,      # Don't do any operations?
-           'redirect-output' => \$redirect,   # Redirect output?
+           'logfile=s'       => \$logfile,   # Redirect output?
            ) or pod2usage( 2 );
 
@@ -51,12 +51,10 @@
 $ipprc->define_camera($camera);
 
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
-    &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
-$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logfile) if $logfile;
 
 # Look for programs we need
 my $missing_tools;
 my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
+my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
Index: branches/bills_081204/ippScripts/scripts/magic_process.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/magic_process.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/magic_process.pl	(revision 20890)
@@ -30,6 +30,6 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect);
-my $skycellroot;
+my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+
 GetOptions(
            'magic_id=s'      => \$magic_id,   # Magic identifier
@@ -42,6 +42,5 @@
            'no-update'       => \$no_update,  # Don't update the database?
            'no-op'           => \$no_op,      # Don't do any operations?
-           'redirect-output' => \$redirect,   # Redirect output?
-           'skycellroot=s'   => \$skycellroot, # root of the warps to find skycells (temporary)
+           'logfile=s'       => \$logfile,
            ) or pod2usage( 2 );
 
@@ -56,5 +55,5 @@
 $ipprc->define_camera($camera);
 
-# Remove streaks doesn't know about nebulous. It expects to be able to append strings to outroot
+# RemoveStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
 # to form valid file names.
 # So forbid nebulous path in outroot. We could relax this by change RemoveStreaks to take all
@@ -68,13 +67,10 @@
 $outroot = $ipprc->file_resolve($outroot);
     
-
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
-    &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
-$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logfile) if $logfile;
 
 # Look for programs we need
 my $missing_tools;
-my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
+my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $removestreaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -104,10 +100,14 @@
 
 
-my @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
+my @outputs;
 ### Do the heavy lifting
 {
-    my $mode;
     my $command;                # Command to execute
-    $command = "$streaks --outroot $outroot";
+    $command = "$removestreaks --outroot $outroot";
+    $command .= " --verbose" if $verbose;
+
+#    $command .= " -u 6";
+#    $command .= " --test";
+
     if (scalar @$inputs == 1 and $node ne "root") {
         #
@@ -115,14 +115,28 @@
         #
         # Leaf node: 'detect' stage
-        $mode = 'detect';
         my $innode = $$inputs[0];     # Input node
 
+        # expected outputs for detect stage
+        @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
+
+        my $diff_id = $innode->{diff_id};
+        if (!$diff_id) {
+            &my_die("input for node has null diff_id", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+        }
         my ($image, $mask, $weight) = resolve_inputs($innode);
 
         $command .= " --detect --image $image --mask $mask --weight $weight";
 
+        # set threshold to 4 sigma
+        $command .= ' -t 4';
+
         # create the list of inputs used at this stage. At higher levels the
-        # thes files will get catenated together to create the file for the subsquent stage
+        # these files will get catenated together to create the file for the subsquent stage
+        # this causes major file pollution, but avoids multi-level queries
+        # at higher level nodes.
         
+        my ($in_fh, $input_list)  = open_list_file($outroot, "input.list");
+        print $in_fh "$outroot\n";
+        close $in_fh;
         my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
         print $ifh "$image\n";
@@ -134,15 +148,4 @@
         print $wfh "$weight\n";
         close $wfh;
-
-        # work around missing wcs in difference images. Use the skycells from warp stage
-        my ($sfh, $skycell_list);
-        if ($skycellroot) {
-           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
-            # at this level the skycell_id is equal to the node
-            my $skycell_id = $node;
-            my $skycell = $ipprc->file_resolve("${skycellroot}.${skycell_id}.skycell");
-            print $sfh "$skycell\n";
-            close $sfh;
-        }
     } else {
         #
@@ -152,43 +155,42 @@
 
         # Branch node: 'merge' stage
-        $mode = 'merge';
-
-        my ($infh, $input_list) = open_list_file($outroot, "input.list");
-        my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
-        my ($mfh, $mask_list)   = open_list_file($outroot, "mask.list");
-        my ($wfh, $weight_list) = open_list_file($outroot, "weight.list");
-        my ($sfh, $skycell_list);
-        if ($skycellroot) {
-           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
-        }
+
+        # expected outputs from merge stage
+        @outputs = ("${outroot}.streaks");
+
+        my ($in_fh, $input_list) = open_list_file($outroot, "input.list");
+        my ($ifh, $image_list)   = open_list_file($outroot, "image.list");
+        my ($mfh, $mask_list)    = open_list_file($outroot, "mask.list");
+        my ($wfh, $weight_list)  = open_list_file($outroot, "weight.list");
+        my ($sfh, $streaks_list) = open_list_file($outroot, "streaks.list");
 
         # do this in eval so we can fault the exposure without
         # passing the $magic_id and $node everywhere
+        # XXX  (they're global dummy fix this)
         eval {
             foreach my $innode (@$inputs) {
                 # root for inputs from previous stage
                 my $in_uri = $innode->{uri};
-                print $infh "$in_uri\n";
-
-                # build image lists by combining the lists from
+                print $sfh "$in_uri\n";
+
+                cat_list_to_list($in_fh, $in_uri, "input.list");
+                # build input lists by combining the lists from
                 # previous stages
                 cat_list_to_list($ifh, $in_uri, "image.list");
                 cat_list_to_list($mfh, $in_uri, "mask.list");
                 cat_list_to_list($wfh, $in_uri, "weight.list");
-                cat_list_to_list($sfh, $in_uri, "wcs.list") if $skycell_list;
             }
-            close $infh;
+            close $in_fh;
             close $ifh;
             close $mfh;
             close $wfh;
-            close $sfh if $skycell_list;
+            close $sfh;
 
             $command .= " --merge --inputs $input_list";
             $command .= " --images $image_list --masks $mask_list --weights $weight_list" ;
-            $command .= " --wcsList $skycell_list" if $skycell_list;
+            $command .= " --inputstreaks $streaks_list";
         };
         if ($@) {
-            &my_die("failed to create file lists: $@", $PS_EXIT_UNKNOWN_ERROR, $magic_id, $node,
-                $PS_EXIT_UNKNOWN_ERROR);
+            &my_die("failed to create file lists: $@", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
         }
     }
@@ -214,5 +216,4 @@
     }
 }
-
 
 
@@ -237,4 +238,36 @@
     }
 }
+if ($node eq "root") {
+    my $streaks_file = "$outroot.streaks";
+    my $resolved = $ipprc->file_resolve($streaks_file);
+
+    my $fh;
+    open $fh, "<$resolved" or 
+        &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    # the first line in the streaks file contains the number of streaks found
+    my $num_streaks = <$fh>;
+    chomp $num_streaks;
+    close $fh;
+    print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose;
+    
+    my $command = "$magictool -addmask";
+    $command   .= " -magic_id $magic_id";
+    $command   .= " -uri $streaks_file";
+    $command   .= " -streaks $num_streaks";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            # This isn't going to work because our result was already entered.
+            &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
 
 
@@ -254,21 +287,25 @@
 }
 
-sub cat_list_to_list   { # ($infh, $in_uri, "input.list");
+sub cat_list_to_list   {
     my $out = shift;        # output file handle
     my $uri = shift;        # uri to append ...
-    my $extension = shift;  # extension to
+    my $extension = shift;  # ... the extension to
+
     my $filename = "$uri.$extension";
 
     my $in;
     open $in, "<$filename" or die "failed to open list file: $filename";
-    my $contents = <$in>;
-    print $out $contents;
+    foreach my $line (<$in>) {
+        print $out $line;
+    }
 }
 
 sub resolve_inputs {
+
     my $node = shift;
     my $input_base = $node->{path_base};
+
     my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
-    my $mask  = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
+    my $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
     my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.WEIGHT", $input_base));
 
@@ -302,10 +339,3 @@
 }
 
-END {
-    my $status = $?;
-    system("sync") == 0
-        or die "failed to execute sync: $!" ;
-    $? = $status;
-}
-
 __END__
Index: branches/bills_081204/ippScripts/scripts/magic_tree.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/magic_tree.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/magic_tree.pl	(revision 20890)
@@ -34,5 +34,5 @@
 # Parse the command-line arguments
 my ($magic_id, $warp_id, $tess_id, $camera, $ra0, $dec0, $dbname, $outroot,
-    $save_temps, $verbose, $no_update, $no_op, $redirect);
+    $save_temps, $verbose, $no_update, $no_op, $logfile);
 GetOptions(
            'magic_id=s'      => \$magic_id,   # Magic identifier
@@ -48,12 +48,11 @@
            'no-update'       => \$no_update,  # Don't update the database?
            'no-op'           => \$no_op,      # Don't do any operations?
-           'redirect-output' => \$redirect,   # Redirect output?
+           'logfile=s'       => \$logfile,   # Redirect output?
            ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_id --warp_id --camera --tess_id --ra --dec --outroot",
+pod2usage( -msg => "Required options: --magic_id --camera --tess_id --ra --dec --outroot",
            -exitval => 3) unless
     defined $magic_id and
-    defined $warp_id and
     defined $tess_id and
     defined $ra0 and
@@ -64,7 +63,5 @@
 $ipprc->define_camera($camera);
 
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
-    &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
-$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logfile) if $logfile;
 
 # Look for programs we need
@@ -80,6 +77,8 @@
 
 ### Get a list of warpSkyfiles
+### This is a workaround to allow processing with older diffSkyfiles that don't have
+### a wcs. If warp_id is provided, we get the wcs from there
 my %warpSkyfiles;                   # hash of warps
-{
+if ($warp_id) {
     my $command = "$warptool -warped -warp_id $warp_id"; # Command to run
     $command .= " -dbname $dbname" if defined $dbname;
@@ -123,5 +122,5 @@
 
     foreach my $input ( @$inputs ) {
-        push @skycells, $input->{node}; # NB: Storing the skycell_id in magicInputSkyfile.node
+        push @skycells, $input; # NB: Storing the skycell_id in magicInputSkyfile.node
     }
 }
@@ -129,5 +128,5 @@
 ### For each skycell, project centre of skycell onto tangent plane of boresight
 my @fields;
-foreach my $skycell_id ( @skycells ) {
+foreach my $input ( @skycells ) {
     # the filename method doesn't add the $skycell_id
 
@@ -139,15 +138,50 @@
 #    $ipprc->skycell_file($tess_id, $skycell_id, $skyfile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR);
 
-    my $warp = $warpSkyfiles{$skycell_id};
-    die "warpSkyfile for $skycell_id not found" if !$warp;
-    my $skyfileBase = $warp->{path_base};
-    my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skyfileBase);
+    my $skyfile;
+    my $skycell_id = $input->{node};
+    if ($warp_id) {
+        # Applying the workaround
+        my $warp = $warpSkyfiles{$skycell_id};
+        die "warpSkyfile for $skycell_id not found" if !$warp;
+        my $skyfileBase = $warp->{path_base};
+        $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skyfileBase);
+    } else {
+        # using the diffSkyfile
+        $skyfile = $input->{uri};
+    }
     my $skyfileResolved = $ipprc->file_resolve( $skyfile );
-    my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved );
+
+#    my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved );
+#    &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+
+    my ($header, $status) = (undef, 0);
+    my $fits =  Astro::FITS::CFITSIO::open_file( $skyfileResolved, READONLY, $status ); 
+    &my_die("failed to open skycell file: $skyfileResolved: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+
+    ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $fits );
+    
     &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
 
+
     # Get the useful header keywords
-    my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR);
+#    my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $naxis1 = $$header{'NAXIS1'};
+    my $naxis2;
+    if ($naxis1) {
+        $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR);
+    } else {
+        # if the skyfile is compressed then the WCS won't be in the primary header, move to the 
+        # extension
+        my $hdutype;
+        $fits->movrel_hdu(1, $hdutype, $status);
+        &my_die("Unable to movrel_hdu: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+
+        ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $fits );
+        &my_die("Unable to read extension header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+        my $xtension = $$header{'XTENSION'} or &my_die("Can't find XTENSION", $magic_id, $PS_EXIT_SYS_ERROR);
+        &my_die("XTENSION found: $xtension", $magic_id, $PS_EXIT_SYS_ERROR) if $xtension ne "'BINTABLE'";
+        $naxis1 = $$header{'ZNAXIS1'} or &my_die("Can't find ZNAXIS1", $magic_id, $PS_EXIT_SYS_ERROR);
+        $naxis2 = $$header{'ZNAXIS2'} or &my_die("Can't find ZNAXIS2", $magic_id, $PS_EXIT_SYS_ERROR);
+    }
     my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR);
     my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR);
@@ -319,4 +353,9 @@
     my $contents = $node->{contents} or die "Can't find contents of node."; # Contents of node
 
+    if ($position eq 'root' and scalar @$contents <= 4) {
+        # don't need to divide, but do we need to sort?
+        return;
+    }
+
     my ($lower, $upper) = divide_list($contents, 'xi');
 
Index: branches/bills_081204/ippScripts/scripts/warp_overlap.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/warp_overlap.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/warp_overlap.pl	(revision 20890)
@@ -61,4 +61,6 @@
 }
 
+&my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL";
+
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
@@ -81,5 +83,5 @@
 }
 
-# Where do we get the astrometry source from? 
+# Where do we get the astrometry source from?
 my $astromSource;               # The astrometry source filerule (eg, PSASTRO.OUTPUT, PSASTRO.OUTPUT.MEF)
 my $astromAccept;               # Accept the astrometry unconditionally?
@@ -116,19 +118,19 @@
     my $astromFile = $ipprc->filename($astromSource, $camRoot); # Astrometry file
     if (!$astromFile) {
-	&my_die("Unable to determine the astrometry source", $warp_id, $PS_EXIT_DATA_ERROR);
+        &my_die("Unable to determine the astrometry source", $warp_id, $PS_EXIT_DATA_ERROR);
     }
     $astromFile = $ipprc->file_resolve($astromFile);
     if (!$astromFile) {
-	&my_die("Unable to resolve real astrometry source filename", $warp_id, $PS_EXIT_DATA_ERROR);
+        &my_die("Unable to resolve real astrometry source filename", $warp_id, $PS_EXIT_DATA_ERROR);
     }
 
     my @matchlist = get_overlaps($astromFile, $tess_dir_abs, $astromAccept); # List of overlaps
     if (! @matchlist) {
-	&my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR);
+        &my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR);
     }
     # Match each of the imfiles to this list (the input images may be split, but the astrometry is not)
     # tess_dir (not tess_dir_abs) is supplied here so it may be written to the db
     foreach my $imfile (@$imfiles) {
-	extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_dir, \@overlaps, \%unique_skycells);
+        extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_dir, \@overlaps, \%unique_skycells);
     }
 } else {
@@ -158,5 +160,5 @@
     print $overlapFile "  skycell_id     STR    $overlap->{skycell_id}\n";
     # XXX convert tess_id here to tess_dir when db scheme is updated
-    print $overlapFile "  tess_id        STR    $overlap->{tess_dir}\n"; 
+    print $overlapFile "  tess_id        STR    $overlap->{tess_dir}\n";
     print $overlapFile "  cam_id         S32    $overlap->{cam_id}\n";
     print $overlapFile "  class_id       STR    $overlap->{class_id}\n";
@@ -249,6 +251,6 @@
 
     if (lc($fileLevel) eq "chip") {
-	# in the case of SPLIT images, all CLASSes are included in the output list
-	# we need to pull out the single CLASS_ID for this imfile from the full list
+        # in the case of SPLIT images, all CLASSes are included in the output list
+        # we need to pull out the single CLASS_ID for this imfile from the full list
         my $class_id = $imfile->{class_id};
         my $chipRoot = $imfile->{chip_path_base};
@@ -258,5 +260,5 @@
         print STDERR "entry: $entry, class: $class_id, extname: $extname, chiproot: $chipRoot\n" if $verbose;
     } else {
-	# in the case of MEF or SINGLE images, there is only a single CLASS in the output list
+        # in the case of MEF or SINGLE images, there is only a single CLASS in the output list
         $entry = $filename;
         print STDERR "entry: $entry\n" if $verbose;
Index: branches/bills_081204/ippScripts/scripts/warp_skycell.pl
===================================================================
--- branches/cnb_branch_20081104/ippScripts/scripts/warp_skycell.pl	(revision 20530)
+++ branches/bills_081204/ippScripts/scripts/warp_skycell.pl	(revision 20890)
@@ -138,9 +138,14 @@
 foreach my $imfile (@$imfiles) {
     my $image = $imfile->{uri}; # Image name
-    my $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
     my $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
 
+    my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); # Mask name
+    if (!$ipprc->file_exists($mask)) {
+	print "Celestial mask $mask not found, using basic mask\n";
+	$mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
+	&my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask);
+    }
+
     &my_die("Couldn't find input file: $image", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image);
-    &my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask);
 
     # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file
Index: branches/bills_081204/ippTasks/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ippTasks/Makefile.am	(revision 20530)
+++ branches/bills_081204/ippTasks/Makefile.am	(revision 20890)
@@ -34,5 +34,7 @@
 	simtest.detverify.auto \
 	simtest.flatcorr.config \
-	simtest.flatcorr.auto
+	simtest.flatcorr.auto \
+	simtest.stack.config \
+	simtest.stack.auto
 
 pantasksdir = $(datadir)/pantasks
Index: branches/bills_081204/ippTasks/magic.pro
===================================================================
--- branches/cnb_branch_20081104/ippTasks/magic.pro	(revision 20530)
+++ branches/bills_081204/ippTasks/magic.pro	(revision 20890)
@@ -10,10 +10,10 @@
 book init magicToTree
 book init magicToProcess
-book init magicToMask
+book init magicToDS
 
 ### Database lists
-$magicTree_DB = 0
-$magicProcess_DB = 0
-$magicMask_DB = 0
+$magicToTree_DB = 0
+$magicToProcess_DB = 0
+$magicToDS_DB = 0
 
 ### Check status of tasks
@@ -21,12 +21,12 @@
   book listbook magicToTree
   book listbook magicToProcess
-  book listbook magicToMask
+  book listbook magicToDS
 end
 
 ### Reset tasks
 macro magic.reset
-  book init magicTree
-  book init magicProcess
-  book init magicMask
+  book init magicToTree
+  book init magicToProcess
+  book init magicToDS
 end
 
@@ -45,8 +45,8 @@
     active true
   end
-  task magic.mask.load
-    active true
-  end
-  task magic.mask.run
+  task magic.destreak.load
+    active true
+  end
+  task magic.destreak.run
     active true
   end
@@ -54,5 +54,5 @@
 
 ### Turn tasks off
-macro magic.on
+macro magic.off
   task magic.tree.load
     active false
@@ -67,8 +67,8 @@
     active false
   end
-  task magic.mask.load
-    active false
-  end
-  task magic.mask.run
+  task magic.destreak.load
+    active false
+  end
+  task magic.destreak.run
     active false
   end
@@ -92,8 +92,8 @@
     else
       # save the DB name for the exit tasks
-      option $DB:$magicTree_DB
-      command magictool -totree -limit 20 -dbname $DB:$magicTree_DB
-      $magicTree_DB ++
-      if ($magicTree_DB >= $DB:n) set magicTree_DB = 0
+      option $DB:$magicToTree_DB
+      command magictool -totree -limit 20 -dbname $DB:$magicToTree_DB
+      $magicToTree_DB ++
+      if ($magicToTree_DB >= $DB:n) set magicToTree_DB = 0
     end
   end
@@ -102,11 +102,11 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout magicTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout magicToTree -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook magicTree
+      book listbook magicToTree
     end
 
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicTree
+    process_cleanup magicToTree
   end
 
@@ -128,21 +128,21 @@
 
   task.exec
-    book npages magicTree -var N
+    book npages magicToTree -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     
     # look for new images (pantaskState == INIT)
-    book getpage magicTree 0 -var pageName -key pantaskState INIT
+    book getpage magicToTree 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword magicTree $pageName pantaskState RUN
-    book getword magicTree $pageName magic_id -var MAGIC_ID
-    book getword magicTree $pageName exp_id -var EXP_ID
-    book getword magicTree $pageName camera -var CAMERA
-    book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE
-    book getword magicTree $pageName dbname -var DBNAME
-    book getword magicTree $pageName tess_id -var TESS_DIR
-    book getword magicTree $pageName ra -var RA
-    book getword magicTree $pageName decl -var DEC
+    book setword magicToTree $pageName pantaskState RUN
+    book getword magicToTree $pageName magic_id -var MAGIC_ID
+    book getword magicToTree $pageName exp_id -var EXP_ID
+    book getword magicToTree $pageName camera -var CAMERA
+    book getword magicToTree $pageName workdir -var WORKDIR_TEMPLATE
+    book getword magicToTree $pageName dbname -var DBNAME
+    book getword magicToTree $pageName tess_id -var TESS_DIR
+    book getword magicToTree $pageName ra -var RA
+    book getword magicToTree $pageName decl -var DEC
 
 #    set.host.for.camera $CAMERA $MAGIC_ID
@@ -152,4 +152,7 @@
 
     basename $TESS_DIR -var TESS_ID
+
+    $TESS_ID = "FIXNS"
+
     sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
 
@@ -181,5 +184,5 @@
   # default exit status
   task.exit    default
-    process_exit magicTree $options:0 $JOB_STATUS
+    process_exit magicToTree $options:0 $JOB_STATUS
   end
 
@@ -187,5 +190,5 @@
   task.exit    timeout
     showcommand timeout
-    book setword magicTree $options:0 pantaskState TIMEOUT
+    book setword magicToTree $options:0 pantaskState TIMEOUT
   end
 end
@@ -208,8 +211,8 @@
     else
       # save the DB name for the exit tasks
-      option $DB:$magicTree_DB
-      command magictool -toprocess -limit 20 -dbname $DB:$magicProcess_DB
-      $magicProcess_DB ++
-      if ($magicProcess_DB >= $DB:n) set magicProcess_DB = 0
+      option $DB:$magicToTree_DB
+      command magictool -toprocess -limit 20 -dbname $DB:$magicToProcess_DB
+      $magicToProcess_DB ++
+      if ($magicToProcess_DB >= $DB:n) set magicToProcess_DB = 0
     end
   end
@@ -218,11 +221,11 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout magicProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout magicToProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook magicProcess
+      book listbook magicToProcess
     end
 
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicProcess
+    process_cleanup magicToProcess
   end
 
@@ -244,19 +247,19 @@
 
   task.exec
-    book npages magicProcess -var N
+    book npages magicToProcess -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     
     # look for new images (pantaskState == INIT)
-    book getpage magicProcess 0 -var pageName -key pantaskState INIT
+    book getpage magicToProcess 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword magicTree $pageName pantaskState RUN
-    book getword magicTree $pageName magic_id -var MAGIC_ID
-    book getword magicTree $pageName exp_id -var EXP_ID
-    book getword magicTree $pageName node -var NODE
-    book getword magicTree $pageName camera -var CAMERA
-    book getword magicTree $pageName workdir -var WORKDIR_TEMPLATE
-    book getword magicTree $pageName dbname -var DBNAME
+    book setword magicToProcess $pageName pantaskState RUN
+    book getword magicToProcess $pageName magic_id -var MAGIC_ID
+    book getword magicToProcess $pageName exp_id -var EXP_ID
+    book getword magicToProcess $pageName node -var NODE
+    book getword magicToProcess $pageName camera -var CAMERA
+    book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
+    book getword magicToProcess $pageName dbname -var DBNAME
 
 #    set.host.for.camera $CAMERA $MAGIC_ID
@@ -295,5 +298,5 @@
   # default exit status
   task.exit    default
-    process_exit magicProcess $options:0 $JOB_STATUS
+    process_exit magicToProcess $options:0 $JOB_STATUS
   end
 
@@ -301,9 +304,9 @@
   task.exit    timeout
     showcommand timeout
-    book setword magicProcess $options:0 pantaskState TIMEOUT
-  end
-end
-
-task	       magic.mask.load
+    book setword magicToProcess $options:0 pantaskState TIMEOUT
+  end
+end
+
+task	       magic.destreak.load
   host         local
 
@@ -314,16 +317,16 @@
 
   stdout NULL
-  stderr $LOGDIR/magic.mask.log
+  stderr $LOGDIR/magic.destreak.log
 
   task.exec
     if ($DB:n == 0)
       option DEFAULT
-      command magictool -tomask -limit 20
+      command magicdstool -todestreak -limit 20
     else
       # save the DB name for the exit tasks
-      option $DB:$magicMask_DB
-      command magictool -tomask -limit 20 -dbname $DB:$magicMask_DB
-      $magicMask_DB ++
-      if ($magicMask_DB >= $DB:n) set magicMask_DB = 0
+      option $DB:$magicToDS_DB
+      command magicdstool -todestreak -limit 20 -dbname $DB:$magicToDS_DB
+      $magicToDS_DB ++
+      if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
     end
   end
@@ -332,11 +335,11 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout magicMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook magicMask
+      book listbook magicToDS
     end
 
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup magicMask
+    process_cleanup magicToDS
   end
 
@@ -352,5 +355,5 @@
 end
 
-task	       magic.mask.run
+task	       magic.destreak.run
   periods      -poll $RUNPOLL
   periods      -exec $RUNEXEC
@@ -358,40 +361,37 @@
 
   task.exec
-    book npages magicMask -var N
+    book npages magicToDS -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     
     # look for new images (pantaskState == INIT)
-    book getpage magicMask 0 -var pageName -key pantaskState INIT
+    book getpage magicToDS 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword magicMask $pageName pantaskState RUN
-    book getword magicMask $pageName magic_id -var MAGIC_ID
-    book getword magicMask $pageName exp_id -var EXP_ID
-    book getword magicMask $pageName camera -var CAMERA
-    book getword magicMask $pageName workdir -var WORKDIR_TEMPLATE
-    book getword magicMask $pageName dbname -var DBNAME
-
-#    set.host.for.camera $CAMERA $MAGIC_ID
-#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+    book setword magicToDS $pageName pantaskState RUN
+    book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID
+    book getword magicToDS $pageName camera -var CAMERA
+    book getword magicToDS $pageName streaks_uri -var STREAKS
+    book getword magicToDS $pageName stage -var STAGE
+    book getword magicToDS $pageName stage_id -var STAGE_ID
+    book getword magicToDS $pageName component -var COMPONENT
+    book getword magicToDS $pageName uri -var URI
+    book getword magicToDS $pageName path_base -var PATH_BASE
+    book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
+    book getword magicToDS $pageName outroot -var OUTROOT
+    book getword magicToDS $pageName recoveryroot -var RECROOT
+    book getword magicToDS $pageName re_place -var REPLACE
+    book getword magicToDS $pageName remove -var REMOVE
+    book getword magicToDS $pageName dbname -var DBNAME
+
+    sprintf logfile "%s/mgcds.%s.%s.%s.log" $OUTROOT $MAGIC_DS_ID $STAGE_ID $COMPONENT
+
     host anyhost
-    $WORKDIR = $WORKDIR_TEMPLATE
-
-    basename $TESS_DIR -var TESS_ID
-    sprintf outroot "%s/%s/%s.mgc.%s.mask" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
-
-    ## generate output log based on filerule (convert the URI to a PATH)
-    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id $MAGIC_ID --basename $outroot`
-    if ("$logfile" == "") 
-      echo "WARNING: logfile not defined in magic.tree.run"
-      break
-    end
-
-    stdout $logfile
-    stderr $logfile
-    dirname $logfile -var outpath
-    mkdir $outpath
-
-    $run = magic_mask.pl --magic_id $MAGIC_ID --camera $CAMERA --outroot $outroot --logfile $logfile
+
+    # TODO: do not add recoveryroot, replace, or remove if they are null or zero
+
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT
+    # --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE 
+    # --logfile $logfile
     add_standard_args run
 
@@ -407,13 +407,19 @@
 
   # default exit status
-  task.exit    default
-    process_exit magicMask $options:0 $JOB_STATUS
+  task.exit    0
+    process_exit magicToDS $options:0 $JOB_STATUS
    end
 
+  # locked list
+  task.exit    default
+    showcommand failure
+    process_exit magicToDS $options:0 $JOB_STATUS
+  end
+
   # operation timed out?
   task.exit    timeout
     showcommand timeout
-    book setword magicMask $options:0 pantaskState TIMEOUT
-  end
-end
-
+    book setword magicToDS $options:0 pantaskState TIMEOUT
+  end
+end
+
Index: branches/bills_081204/ippTasks/simtest.basic.auto
===================================================================
--- branches/cnb_branch_20081104/ippTasks/simtest.basic.auto	(revision 20530)
+++ branches/bills_081204/ippTasks/simtest.basic.auto	(revision 20890)
@@ -70,9 +70,9 @@
 END
 
-### Magic automation???
-automate METADATA
-  name       STR MAGIC
-  regular    STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@"
-END
+###### Magic automation???
+###automate METADATA
+###  name       STR MAGIC
+###  regular    STR "magictool -definebyquery -workdir file://@CWD@/magic -good_frac 0.2 -dbname @DBNAME@"
+###END
 
 
Index: branches/bills_081204/ippTasks/simtest.large.config
===================================================================
--- branches/bills_081204/ippTasks/simtest.large.config	(revision 20890)
+++ branches/bills_081204/ippTasks/simtest.large.config	(revision 20890)
@@ -0,0 +1,73 @@
+
+# define the simulated data to be generated
+SEQUENCE MULTI
+
+SEQUENCE METADATA
+  OBSTYPE    STR BIAS
+  CAMERA     STR SIMTEST
+  NIMAGES    S32  20
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR DARK
+  CAMERA     STR SIMTEST
+  @EXPTIMES  F32  30.0, 300.0
+  @NIMAGES   S32  10,   10
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR FLAT
+  CAMERA     STR SIMTEST
+
+  FILTERS    STR r,r,i
+  @EXPTIMES  F32 0.1,20.0,20.0
+
+  NSETUP     S32 5
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR FLAT
+  CAMERA     STR SIMTEST
+
+  FILTERS    STR r,r,r,r,r
+  @EXPTIMES  F32 0.5,1.0,2.0,5.0,10.0
+
+  NSETUP     S32 1
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR OBJECT
+  CAMERA     STR SIMTEST
+
+### Carina
+  CENTER.RA  F32 270.75
+  CENTER.DEC F32 -23.7
+### COSMOS field
+#  CENTER.RA  F32 150.119167 
+#  CENTER.DEC F32   2.205833
+
+  OFFSET.RA  F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction)
+  OFFSET.DEC F32 300.0 # linear offset in arcsec (do not include cos(DEC) correction)
+  OFFSET.NR  S32 3
+  OFFSET.ND  S32 3
+
+  DITHER.RA  F32 60.0   # linear offset in arcsec (do not include cos(DEC) correction)
+  DITHER.DEC F32 60.0   # linear offset in arcsec (do not include cos(DEC) correction)
+  DITHER.NR  S32 3
+  DITHER.ND  S32 3
+
+  DVODB      STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest 
+  FILTERS    STR r
+  @EXPTIMES  F32 30.00
+  @SKYMAGS   F32 20.86
+
+  # XXX use a more realistic IQ distribution...
+  IQ_MIN     F32 0.55
+  IQ_MAX     F32 1.25
+  
+  # rotation sequence
+  POS_MIN    F32 0.0
+  POS_MAX    F32 0.0
+  POS_DELTA  F32 1.0
+
+END
Index: branches/bills_081204/ippTasks/simtest.stack.auto
===================================================================
--- branches/bills_081204/ippTasks/simtest.stack.auto	(revision 20890)
+++ branches/bills_081204/ippTasks/simtest.stack.auto	(revision 20890)
@@ -0,0 +1,65 @@
+
+automate MULTI
+
+# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK
+# success on CHECK -> LAUNCH
+# failure on BLOCK -> CHECK
+
+# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis
+
+automate METADATA
+  name       STR BIAS
+  check      STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@"
+  ncheck     S32 20
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR DARK
+  check      STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type DARK -dbname @DBNAME@"
+END
+ 
+automate METADATA
+  name       STR SHUTTER
+  check      STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR FLAT-r
+  check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR FLAT-i
+  check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR OBJECT-i
+  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter i -det_type FLAT"
+  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
+  block      STR NONE
+END
+
+automate METADATA
+  name       STR OBJECT-r
+  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter r -det_type FLAT"
+  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc"
+  block      STR NONE
+END
+
+### Stack automation???
+automate METADATA
+  name       STR STACK
+  regular    STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
+END
+
Index: branches/bills_081204/ippTasks/simtest.stack.config
===================================================================
--- branches/bills_081204/ippTasks/simtest.stack.config	(revision 20890)
+++ branches/bills_081204/ippTasks/simtest.stack.config	(revision 20890)
@@ -0,0 +1,76 @@
+
+# define the simulated data to be generated
+SEQUENCE MULTI
+
+SEQUENCE METADATA
+  OBSTYPE    STR BIAS
+  CAMERA     STR SIMTEST
+  NIMAGES    S32  20
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR DARK
+  CAMERA     STR SIMTEST
+  @EXPTIMES  F32  30.0, 300.0
+  @NIMAGES   S32  10,   10
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR FLAT
+  CAMERA     STR SIMTEST
+
+  FILTERS    STR r,r
+  @EXPTIMES  F32 0.1,20.0
+
+  NSETUP     S32 5
+END
+
+SEQUENCE METADATA
+  OBSTYPE    STR FLAT
+  CAMERA     STR SIMTEST
+
+  FILTERS    STR r,r,r,r,r
+  @EXPTIMES  F32 0.5,1.0,2.0,5.0,10.0
+
+  NSETUP     S32 1
+END
+
+
+SEQUENCE METADATA
+### We want both short and long exposures, and dark and bright sky
+  OBSTYPE    STR OBJECT
+  CAMERA     STR SIMTEST
+
+### Carina
+  CENTER.RA  F32 270.75
+  CENTER.DEC F32 -23.7
+### COSMOS field
+#  CENTER.RA  F32 150.119167 
+#  CENTER.DEC F32   2.205833
+
+  OFFSET.RA  F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction)
+  OFFSET.DEC F32 3600.0 # linear offset in arcsec (do not include cos(DEC) correction)
+  OFFSET.NR  S32 1
+  OFFSET.ND  S32 1
+
+  DITHER.RA  F32 10.0   # linear offset in arcsec (do not include cos(DEC) correction)
+  DITHER.DEC F32 10.0   # linear offset in arcsec (do not include cos(DEC) correction)
+  DITHER.NR  S32 3
+  DITHER.ND  S32 3
+
+  DVODB      STR /data/alala.0/ipp/ippRefs/catdir.synth.simtest 
+  FILTERS    STR r,r,r
+  @EXPTIMES  F32 10,300,300
+  @SKYMAGS   F32 21,21,18
+
+  # XXX use a more realistic IQ distribution...
+  IQ_MIN     F32 1.00
+  IQ_MAX     F32 1.00
+  
+  # rotation sequence
+  POS_MIN    F32 0.0
+  POS_MAX    F32 0.0
+  POS_DELTA  F32 1.0
+
+END
+
Index: branches/bills_081204/ippTests/buildtest/ippcheck.003
===================================================================
--- branches/bills_081204/ippTests/buildtest/ippcheck.003	(revision 20890)
+++ branches/bills_081204/ippTests/buildtest/ippcheck.003	(revision 20890)
@@ -0,0 +1,144 @@
+#!/bin/bash
+#automated check of ipp build
+#Bill Giebink 9/08
+
+#write step and return status to summary log file
+writeStatus()
+{
+if [ $2 = "0" ]; then
+  status="PASS"
+else
+  status="FAIL"
+fi
+echo $1$status >> $logfile
+}
+
+#write step to summary, build, and error files
+writeStep()
+{
+  echo $1 >> $logfile
+  echo -e "\n++++"$1"++++" >> $buildfile 
+  echo -e "\n++++"$1"++++" >> $errorfile
+}
+
+PATH=$PATH:/bin:/usr/bin:/usr/ucb:/etc:.
+export PATH
+
+#log files
+buildfile="/data/ipp003.0/ippTests/buildtest/logs/buildlog.$(date -u +%Y%m%d)"
+errorfile="/data/ipp003.0/ippTests/buildtest/logs/errorlog.$(date -u +%Y%m%d)"
+logfile="/data/ipp003.0/ippTests/buildtest/logs/summlog.$(date -u +%Y%m%d)"
+echo "" > $buildfile
+echo "" > $errorfile
+echo " Build file name: "$buildfile > $logfile
+echo "Error file name: "$errorfile >> $logfile
+
+#remove old psconfig
+writeStep "Remove old psconfig dir"
+rm -rf /data/ipp003.0/ippTests/psconfig
+writeStatus "Remove old psconfig dir status: " $?
+
+#remove old ipp build
+writeStep "Remove old ipp build"
+cd /data/ipp003.0/ippTests/src
+rm -rf /data/ipp003.0/ippTests/src/ipp
+writeStatus "Remove old ipp build status: " $?
+
+#cvs checkout
+CVSROOT="giebink@cvs.pan-starrs.ifa.hawaii.edu:/cvsroot/pan-starrs"
+export CVSROOT
+echo "CVSROOT =" $CVSROOT >> $logfile
+writeStep "Begin cvs co ipp" 
+cvs co ipp >> $buildfile 2>> $errorfile 
+writeStatus "CVS ipp checkout status: " $?
+
+#remove old C libraries and Perl modules
+cd /data/ipp003.0/ippTests/src
+writeStep "Remove old C libraries"
+rm -rf /data/ipp003.0/ippTests/src/extlibs
+writeStatus "Remove old extlib dir status: " $?
+
+writeStep "Remove old Perl modules"
+rm -rf /data/ipp003.0/ippTests/src/extperl
+writeStatus "Remove old extperl dir status: " $?
+
+#cvs checkout extlibs
+writeStep "Begin cvs co extlibs" 
+cvs co extlibs>> $buildfile 2>> $errorfile 
+writeStatus "CVS extlibs checkout status: " $?
+
+#cvs checkout extperl
+writeStep "Begin cvs co extperl" 
+cvs co extperl>> $buildfile 2>> $errorfile 
+writeStatus "CVS extperl checkout status: " $?
+
+#psconfig bootstrap
+writeStep "Change to psconfig dir"
+cd /data/ipp003.0/ippTests/src/ipp/psconfig
+writeStatus "Change to psconfig dir status: " $?
+PSCONFDIR=/data/ipp003.0/ippTests/psconfig
+export PSCONFDIR
+echo "PSCONFDIR =" $PSCONFDIR >> $logfile
+
+writeStep "Running psbuild -bootstrap"
+psbuild -bootstrap $PSCONFDIR >> $buildfile 2>> $errorfile
+writeStatus "Run psbuild -bootstrap status: " $?
+
+#psconfig default
+writeStep "Running psconfig default"
+if [ -f /data/ipp003.0/ippTests/psconfig/psconfig.csh ]; then
+  alias psconfig='source /data/ipp003.0/ippTests/psconfig/psconfig.bash'
+else
+  echo "psconfig not available" >> $logfile
+fi
+/data/ipp003.0/ippTests/psconfig/psconfig.bash default >> $buildfile 2>> $errorfile
+writeStatus "Run psconfig default status: " $?
+
+#check C libraries
+writeStep "Running pschecklibs -build"
+pschecklibs -build >> $buildfile 2>> $errorfile
+writeStatus "Run C library check status: " $?
+
+#check perl 
+writeStep "Running pscheckperl -build"
+pscheckperl -build >> $buildfile 2>> $errorfile
+writeStatus "Run Perl check status: " $?
+
+#check psbuild
+writeStep "Running psbuild -profile -dev"
+psbuild -profile -dev >> $buildfile 2>> $errorfile
+writeStatus "Run psbuild status: " $?
+
+#build ipp configuration
+cd /data/ipp003.0/ippTests/src/ipp/ippconfig
+writeStatus "Change to ippconfig dir status: " $?
+
+writeStep "Building ipp configuration: configure --prefix='/data/ipp003.0/ippTests/psconfig/default.lin64'"
+configure --prefix='/data/ipp003.0/ippTests/psconfig/default.lin64' >> $buildfile 2>> $errorfile
+writeStatus "ipp configure status: " $?
+
+writeStep "Building ipp configuration: make"
+cd /data/ipp003.0/ippTests/src/ipp/ippconfig
+make >> $buildfile 2>> $errorfile
+writeStatus "ipp make status: " $?
+
+writeStep "Building ipp configuration: make install"
+cd /data/ipp003.0/ippTests/src/ipp/ippconfig
+make install >> $buildfile 2>> $errorfile
+writeStatus "ipp make install status: " $?
+
+#copy config files
+writeStep "Copying configuration files: site.config, .ptolemyrc, .ipprc"
+cp /data/ipp003.0/ippTests/site.config /data/ipp003.0/ippTests/psconfig/default.lin64/share/ippconfig/site.config >> $buildfile 2>> $errorfile 
+writeStatus "Copy site.config status: " $?
+
+#last two file copies do not work due to the way the ippTests dir is linked in my home dir
+cp /data/ipp003.0/ippTests/src/ipp/ippconfig/dvo.site /netdisks/panstarrs/giebink/.ptolemyrc >> $buildfile 2>> $errorfile
+writeStatus "Copy dvo.site status: " $?
+
+cp /data/ipp003.0/ippTests/src/ipp/ippconfig/ipprc.config /netdisks/panstarrs/giebink/.ipprc >> $buildfile 2>> $errorfile
+writeStatus "Copy ipprc.config status: " $?
+
+#mail results
+writeStep "Mail results"
+/data/ipp003.0/ippTests/buildtest/mailresults.pl
Index: branches/bills_081204/ippTests/buildtest/ippcheck.022
===================================================================
--- branches/cnb_branch_20081104/ippTests/buildtest/ippcheck.022	(revision 20530)
+++ 	(revision )
@@ -1,123 +1,0 @@
-#!/bin/bash
-#automated check of ipp build
-#Bill Giebink 9/08
-#called by cron daily at 19:30 HST
-
-#write step and return status to summary log file
-writeStatus()
-{
-if [ $2 = "0" ]; then
-  status="PASS"
-else
-  status="FAIL"
-fi
-echo $1$status >> $logfile
-}
-
-#write step to summary, build, and error files
-writeStep()
-{
-  echo $1 >> $logfile
-  echo -e "\n++++"$1"++++" >> $buildfile 
-  echo -e "\n++++"$1"++++" >> $errorfile
-}
-
-PATH=$PATH:/bin:/usr/bin:/usr/ucb:/etc:.
-export PATH
-
-#log files
-buildfile="/data/ipp022.0/ippTests/buildtest/logs/buildlog.$(date -u +%Y%m%d)"
-errorfile="/data/ipp022.0/ippTests/buildtest/logs/errorlog.$(date -u +%Y%m%d)"
-logfile="/data/ipp022.0/ippTests/buildtest/logs/summlog.$(date -u +%Y%m%d)"
-echo " Build file name: "$buildfile > $logfile
-echo "Error file name: "$errorfile >> $logfile
-
-#remove old psconfig
-writeStep "Remove old psconfig dir"
-rm -rf /data/ipp022.0/ippTests/psconfig
-writeStatus "Remove old psconfig dir status: " $?
-
-#remove old ipp build
-writeStep "Remove old ipp build"
-cd /data/ipp022.0/ippTests/src
-rm -rf /data/ipp022.0/ippTests/src/ipp
-writeStatus "Remove old ipp build status: " $?
-
-#cvs checkout
-CVSROOT="giebink@cvs.pan-starrs.ifa.hawaii.edu:/cvsroot/pan-starrs"
-export CVSROOT
-echo "CVSROOT =" $CVSROOT >> $logfile
-writeStep "Begin cvs co ipp" 
-cvs co ipp >> $buildfile 2>> $errorfile 
-writeStatus "CVS checkout status: " $?
-
-#psconfig bootstrap
-writeStep "Change to psconfig dir"
-cd /data/ipp022.0/ippTests/src/ipp/psconfig
-writeStatus "Change to psconfig dir status: " $?
-PSCONFDIR=/data/ipp022.0/ippTests/psconfig
-export PSCONFDIR
-echo "PSCONFDIR =" $PSCONFDIR >> $logfile
-
-writeStep "Running psbuild -bootstrap"
-psbuild -bootstrap $PSCONFDIR >> $buildfile 2>> $errorfile
-writeStatus "Run psbuild -bootstrap status: " $?
-
-#psconfig default
-writeStep "Running psconfig default"
-if [ -f /data/ipp022.0/ippTests/psconfig/psconfig.csh ]; then
-  alias psconfig='source /data/ipp022.0/ippTests/psconfig/psconfig.bash'
-else
-  echo "psconfig not available" >> $logfile
-fi
-/data/ipp022.0/ippTests/psconfig/psconfig.bash default >> $buildfile 2>> $errorfile
-writeStatus "Run psconfig default status: " $?
-
-#check C libraries
-writeStep "Running pschecklibs -build"
-pschecklibs -build >> $buildfile 2>> $errorfile
-writeStatus "Run C library check status: " $?
-
-#check perl 
-writeStep "Running pscheckperl -build -force Astro-FITS-CFITSIO"
-pscheckperl -build -force Astro-FITS-CFITSIO >> $buildfile 2>> $errorfile
-writeStatus "Run Perl check status: " $?
-
-#check psbuild
-writeStep "Running psbuild -profile -dev"
-psbuild -profile -dev >> $buildfile 2>> $errorfile
-writeStatus "Run psbuild status: " $?
-
-#build ipp configuration
-cd /data/ipp022.0/ippTests/src/ipp/ippconfig
-writeStatus "Change to ippconfig dir status: " $?
-
-writeStep "Building ipp configuration: configure --prefix='/data/ipp022.0/ippTests/psconfig/default.lin64'"
-configure --prefix='/data/ipp022.0/ippTests/psconfig/default.lin64' >> $buildfile 2>> $errorfile
-writeStatus "ipp configure status: " $?
-
-writeStep "Building ipp configuration: make"
-cd /data/ipp022.0/ippTests/src/ipp/ippconfig
-make >> $buildfile 2>> $errorfile
-writeStatus "ipp make status: " $?
-
-writeStep "Building ipp configuration: make install"
-cd /data/ipp022.0/ippTests/src/ipp/ippconfig
-make install >> $buildfile 2>> $errorfile
-writeStatus "ipp make install status: " $?
-
-#copy config files
-writeStep "Copying configuration files: site.config, .ptolemyrc, .ipprc"
-cp /data/ipp022.0/ippTests/site.config /data/ipp022.0/ippTests/psconfig/default.lin64/share/ippconfig/site.config >> $buildfile 2>> $errorfile 
-writeStatus "Copy site.config status: " $?
-
-#last two file copies do not work due to the way the ippTests dir is linked in my home dir
-cp /data/ipp022.0/ippTests/src/ipp/ippconfig/dvo.site /netdisks/panstarrs/giebink/.ptolemyrc >> $buildfile 2>> $errorfile
-writeStatus "Copy dvo.site status: " $?
-
-cp /data/ipp022.0/ippTests/src/ipp/ippconfig/ipprc.config /netdisks/panstarrs/giebink/.ipprc >> $buildfile 2>> $errorfile
-writeStatus "Copy ipprc.config status: " $?
-
-#mail results
-writeStep "Mail results"
-/data/ipp022.0/ippTests/buildtest/mailresults.pl
Index: branches/bills_081204/ippTests/buildtest/mailresults.pl
===================================================================
--- branches/bills_081204/ippTests/buildtest/mailresults.pl	(revision 20890)
+++ branches/bills_081204/ippTests/buildtest/mailresults.pl	(revision 20890)
@@ -0,0 +1,40 @@
+#!/usr/bin/env perl
+#email results of ipp build check to ipp-dev
+#email sent if build check fails or build summary file not found
+#Bill Giebink 9/08
+
+use strict;
+use warnings;
+
+use Email::Send;
+
+my $email="ps-ipp-dev\@ifa.hawaii.edu";
+
+my $today = `date -u +%Y%m%d`;
+my $logfile = "/data/ipp003.0/ippTests/buildtest/logs/summlog.$today";
+
+my @logcontents;
+
+#check existence of log file
+if (open (FPTR, $logfile)) {
+  @logcontents = <FPTR>;
+  } else {
+  @logcontents = "FAIL: Can't Open File: $logfile\n";
+}
+close (FPTR);
+
+my $message =<<END;
+To: $email
+From: giebink\@ifa.hawaii.edu
+Subject: ipp build check summary $today
+
+@logcontents
+
+END
+
+#only send message if check FAILs
+if (grep {/FAIL/} @logcontents) {
+  my $sender = Email::Send->new({mailer => 'SMTP'});
+  $sender->mailer_args([Host => 'hale.ifa.hawaii.edu']);
+  $sender->send($message);
+}
Index: branches/bills_081204/ippTools/share/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/Makefile.am	(revision 20530)
+++ branches/bills_081204/ippTools/share/Makefile.am	(revision 20890)
@@ -101,10 +101,8 @@
      magictool_definebyquery.sql \
      magictool_definebyquery_insert.sql \
-     magictool_definebyquery_select_part1.sql \
-     magictool_definebyquery_select_part2.sql \
+     magictool_definebyquery_select.sql \
      magictool_definebyquery_select_test.sql \
      magictool_definebyquery_temp_create.sql \
      magictool_definebyquery_temp_insert.sql \
-     magictool_definebyquery_temp_insert_groupby.sql \
      magictool_inputs.sql \
      magictool_inputskyfile.sql \
@@ -115,4 +113,12 @@
      magictool_toskyfilemask.sql \
      magictool_totree.sql \
+     magictool_diffskyfile.sql \
+     magictool_warpskyfile.sql \
+     magictool_chipprocessedimfile.sql \
+     magictool_rawimfile.sql \
+     magicdstool_todestreak.sql \
+     magicdstool_completed_runs.sql \
+     magicdstool_getskycells.sql \
+     magicdstool_getrunids.sql \
      pstamptool_addjob_otherjob.sql \
      pstamptool_addjob_stampjob.sql \
@@ -159,5 +165,6 @@
      warptool_pendingcleanuprun.sql \
      warptool_pendingcleanupskyfile.sql \
-     warptool_revertwarped.sql \
+     warptool_revertwarped_update.sql \
+     warptool_revertwarped_delete.sql \
      warptool_scmap.sql \
      warptool_tooverlap.sql \
Index: branches/bills_081204/ippTools/share/chiptool_completely_processed_exp.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/chiptool_completely_processed_exp.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/chiptool_completely_processed_exp.sql	(revision 20890)
@@ -13,5 +13,5 @@
     end_stage
 FROM
-    (SELECT 
+    (SELECT
         chipRun.*,
         rawImfile.class_id as rawimfile_class_id,
@@ -19,5 +19,6 @@
     FROM chipRun
     JOIN rawImfile
-        USING(exp_id)
+        ON chipRun.exp_id = rawImfile.exp_id
+        AND rawImfile.ignored = 0
     LEFT JOIN chipProcessedImfile
         ON chipRun.chip_id = chipProcessedImfile.chip_id
Index: branches/bills_081204/ippTools/share/chiptool_pendingimfile.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/chiptool_pendingimfile.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/chiptool_pendingimfile.sql	(revision 20890)
@@ -12,5 +12,6 @@
     USING(exp_id)
 JOIN rawImfile
-    USING(exp_id)
+    ON rawExp.exp_id = rawImfile.exp_id
+    AND rawImfile.ignored = 0
 LEFT JOIN chipProcessedImfile
     ON chipRun.chip_id = chipProcessedImfile.chip_id
Index: branches/bills_081204/ippTools/share/difftool_definebyquery.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/difftool_definebyquery.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/difftool_definebyquery.sql	(revision 20890)
@@ -7,10 +7,7 @@
     warpsToDiff.tess_id,
     warpsToDiff.filter,
-    stacksForDiff.stack_label,
-    warpsToDiff.warp_label,
     warpsToDiff.good_frac,
     warpsToDiff.diff_id,
     warpsToDiff.kind,
-    warpsToDiff.label,
     current_stack_id,
     best_stack_id
@@ -46,4 +43,5 @@
         warpSkyfile.fault = 0
         AND warpSkyfile.ignored = 0
+    -- warpsToDiff WHERE hook %s
     ) AS warpsToDiff
 -- Get best stack as a function of skycell_id, filter
@@ -52,11 +50,10 @@
         MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment
         skycell_id,
-        tess_id,
-        filter,
-	label as stack_label
+        filter
     FROM stackRun
     JOIN stackSumSkyfile USING(stack_id)
     WHERE stackRun.state = 'full'
         AND stackSumSkyfile.fault = 0
+    -- stacksForDiff WHERE hook %s
     GROUP BY
         skycell_id,
Index: branches/bills_081204/ippTools/share/difftool_skyfile.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/difftool_skyfile.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/difftool_skyfile.sql	(revision 20890)
@@ -18,2 +18,12 @@
     ON diffInputSkyfile.diff_id = diffRun.diff_id
     AND diffInputSkyfile.template = 0
+JOIN warpRun
+    USING(warp_id)
+JOIN fakeRun
+    USING(fake_id)
+JOIN camRun
+    USING(cam_id)
+JOIN chipRun
+    USING(chip_id)
+JOIN rawExp
+    USING(exp_id)
Index: branches/bills_081204/ippTools/share/faketool_completely_processed_exp.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/faketool_completely_processed_exp.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/faketool_completely_processed_exp.sql	(revision 20890)
@@ -13,5 +13,5 @@
     epoch
 FROM
-    (SELECT 
+    (SELECT
         fakeRun.*,
         rawImfile.class_id as rawimfile_class_id,
@@ -25,5 +25,6 @@
         USING(exp_id)
     JOIN rawImfile
-        USING(exp_id)
+        ON rawImfile.exp_id = rawExp.exp_id
+        AND rawImfile.ignored = 0
     LEFT JOIN fakeProcessedImfile
         ON fakeRun.fake_id = fakeProcessedImfile.fake_id
Index: branches/bills_081204/ippTools/share/magicdstool_completed_runs.sql
===================================================================
--- branches/bills_081204/ippTools/share/magicdstool_completed_runs.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magicdstool_completed_runs.sql	(revision 20890)
@@ -0,0 +1,86 @@
+SELECT DISTINCT
+    magic_ds_id
+FROM
+    (
+-- raw stage
+SELECT 
+    magicDSRun.magic_ds_id
+    FROM magicDSRun
+    JOIN rawImfile ON stage_id = rawImfile.exp_id
+    LEFT JOIN magicDSFile
+        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+        AND magicDSFile.component = rawImfile.class_id
+    WHERE
+        magicDSRun.state = 'run'
+        AND magicDSRun.stage = 'raw'
+    GROUP BY
+        magicDSRun.magic_ds_id,
+        rawImfile.exp_id
+    HAVING
+        COUNT(rawImfile.class_id) = COUNT(magicDSFile.component)
+        AND SUM(magicDSFile.fault) = 0
+UNION
+-- chip stage
+SELECT 
+    magicDSRun.magic_ds_id
+    FROM magicDSRun
+    JOIN chipProcessedImfile ON stage_id = chip_id
+    LEFT JOIN magicDSFile
+        ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
+        AND magicDSFile.component = chipProcessedImfile.class_id
+    WHERE
+        magicDSRun.state = 'run'
+        AND magicDSRun.stage = 'chip'
+    GROUP BY
+        magic_ds_id,
+        chip_id
+    HAVING
+        COUNT(chipProcessedImfile.class_id) = COUNT(magicDSFile.component)
+        AND SUM(magicDSFile.fault) = 0
+UNION
+-- warp stage
+SELECT 
+    magicDSRun.magic_ds_id
+    FROM magicDSRun
+    JOIN warpSkyfile on stage_id = warp_id
+    LEFT JOIN magicDSFile
+        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+        AND magicDSFile.component = warpSkyfile.skycell_id
+    WHERE
+        magicDSRun.state = 'run'
+        AND magicDSRun.stage = 'warp'
+        AND warpSkyfile.fault = 0
+        AND warpSkyfile.ignored = 0
+    GROUP BY
+        magicDSRun.magic_ds_id,
+        warp_id
+    HAVING
+        COUNT(warpSkyfile.skycell_id) = COUNT(magicDSFile.component)
+        AND SUM(magicDSFile.fault) = 0
+UNION
+-- diff stage
+SELECT DISTINCT
+    magicDSRun.magic_ds_id
+    FROM magicDSRun
+    JOIN magicRun USING (magic_id)
+    JOIN magicInputSkyfile USING(magic_id)
+    -- Do we really need to join back to diffInputSkyfile here?
+    JOIN diffRun USING(diff_id)
+    JOIN diffSkyfile USING(diff_id)
+    JOIN diffInputSkyfile USING(diff_id)
+    LEFT JOIN magicDSFile
+        ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+        AND magicDSFile.component = diffInputSkyfile.skycell_id
+    WHERE
+        magicDSRun.state = 'run'
+        AND magicDSRun.stage = 'diff'
+        AND diffRun.state = 'full'
+        AND diffSkyfile.fault = 0
+    GROUP BY
+        magicDSRun.magic_ds_id,
+        magicRun.magic_id
+    HAVING
+        COUNT(magicInputSkyfile.node) = COUNT(magicDSFile.component)
+        AND SUM(magicDSFile.fault) = 0
+
+   ) as Foo
Index: branches/bills_081204/ippTools/share/magicdstool_getrunids.sql
===================================================================
--- branches/bills_081204/ippTools/share/magicdstool_getrunids.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magicdstool_getrunids.sql	(revision 20890)
@@ -0,0 +1,16 @@
+-- find the inputs for a new magicDSRun from an existing magicRun
+SELECT DISTINCT
+    magic_id,
+    magicRun.exp_id,
+    chip_id,
+    cam_id,
+    warp_id 
+FROM magicRun 
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffInputSkyfile USING(diff_id)
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+WHERE template  = 0
+AND magic_id = %ld
Index: branches/bills_081204/ippTools/share/magicdstool_getskycells.sql
===================================================================
--- branches/bills_081204/ippTools/share/magicdstool_getskycells.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magicdstool_getskycells.sql	(revision 20890)
@@ -0,0 +1,27 @@
+SELECT DISTINCT
+    diffSkyfile.diff_id,
+    diffRun.skycell_id,
+    diffSkyfile.uri,
+    diffSkyfile.path_base
+FROM magicDSRun
+JOIN magicRun USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile
+    ON diffInputSkyfile.diff_id = diffRun.diff_id
+    AND diffInputSkyfile.skycell_id = diffRun.skycell_id
+    -- Want input warps only
+    AND diffInputSkyfile.warp_id IS NOT NULL
+    AND diffInputSkyfile.template = 0
+JOIN warpSkyCellMap
+    ON warpSkyCellMap.warp_id = diffInputSkyfile.warp_id
+    AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id
+JOIN warpSkyfile
+    ON warpSkyfile.warp_id = warpSkyCellMap.warp_id
+    AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id
+    AND warpSkyfile.ignored = 0
+WHERE
+    diffRun.state = 'full'
+    AND diffSkyfile.fault = 0
+    AND magic_ds_id = %lld
Index: branches/bills_081204/ippTools/share/magicdstool_todestreak.sql
===================================================================
--- branches/bills_081204/ippTools/share/magicdstool_todestreak.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magicdstool_todestreak.sql	(revision 20890)
@@ -0,0 +1,134 @@
+SELECT *
+FROM (
+SELECT DISTINCT
+    magicDSRun.magic_ds_id,
+    magicRun.magic_id,
+    magicRun.exp_id,
+    camera,
+    magicMask.uri as streaks_uri,
+    stage,
+    stage_id,
+    class_id as component,
+    rawImfile.uri as uri,
+    NULL as path_base,
+    camProcessedExp.path_base as cam_path_base,
+    outroot,
+    recoveryroot,
+    re_place,
+    remove
+FROM magicDSRun
+JOIN magicMask USING (magic_id)
+JOIN magicRun USING (magic_id)
+JOIN camProcessedExp USING(cam_id)
+JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id
+LEFT JOIN magicDSFile
+    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+    AND magicDSFile.component = rawImfile.class_id
+WHERE
+    magicDSRun.state = 'run'
+    AND magicDSRun.stage = 'raw'
+    AND magicDSFile.component IS NULL
+UNION
+  -- chipProcesedImfiles
+SELECT DISTINCT
+    magicDSRun.magic_ds_id,
+    magicDSRun.magic_id,
+    chipRun.exp_id,
+    camera,
+    magicMask.uri as streaks_uri,
+    stage,
+    stage_id,
+    class_id as component,
+    chipProcessedImfile.uri,
+    chipProcessedImfile.path_base,
+    camProcessedExp.path_base as cam_path_base,
+    outroot,
+    recoveryroot,
+    re_place,
+    remove
+FROM magicDSRun
+JOIN magicMask USING (magic_id)
+JOIN camProcessedExp USING(cam_id)
+JOIN chipRun ON chip_id = stage_id
+JOIN chipProcessedImfile USING(chip_id)
+JOIN rawExp ON chipRun.exp_id = rawExp.exp_id
+LEFT JOIN magicDSFile
+    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+    AND magicDSFile.component = chipProcessedImfile.class_id
+WHERE
+    magicDSRun.state = 'run'
+    AND magicDSRun.stage = 'chip'
+    AND chipRun.state = 'full'
+    AND chipProcessedImfile.fault = 0
+    AND magicDSFile.component IS NULL
+UNION
+-- warpSkyfiles
+SELECT DISTINCT
+    magicDSRun.magic_ds_id,
+    magicRun.magic_id,
+    magicRun.exp_id,
+    camera,
+    magicMask.uri as streaks_uri,
+    stage,
+    stage_id,
+    warpSkyfile.skycell_id as component,
+    warpSkyfile.uri,
+    warpSkyfile.path_base,
+    NULL as cam_path_base,
+    outroot,
+    recoveryroot,
+    re_place,
+    remove
+FROM magicDSRun
+JOIN magicMask USING (magic_id)
+JOIN magicRun USING (magic_id)
+JOIN magicInputSkyfile USING(magic_id)
+JOIN warpSkyfile ON warp_id = stage_id
+JOIN warpRun USING(warp_id)
+JOIN rawExp ON magicRun.exp_id = rawExp.exp_id
+LEFT JOIN magicDSFile
+    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+    AND magicDSFile.component = warpSkyfile.skycell_id
+WHERE
+    magicDSRun.state = 'run'
+    AND magicDSRun.stage = 'warp'
+    AND warpRun.state = 'full'
+    AND warpSkyfile.fault = 0
+    AND warpSkyfile.ignored = 0
+    AND magicDSFile.component IS NULL
+UNION
+-- diffSkyfiles
+SELECT DISTINCT
+    magicDSRun.magic_ds_id,
+    magicRun.magic_id,
+    magicRun.exp_id,
+    rawExp.camera,
+    magicMask.uri as streaks_uri,
+    stage,
+    diff_id as stage_id,
+    diffInputSkyfile.skycell_id as component,
+    diffSkyfile.uri,
+    diffSkyfile.path_base,
+    NULL as cam_path_base,
+    outroot,
+    recoveryroot,
+    re_place,
+    remove
+FROM rawExp
+JOIN magicRun USING (exp_id)
+JOIN magicMask USING (magic_id)
+JOIN magicDSRun USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id)
+LEFT JOIN magicDSFile
+    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+    AND magicDSFile.component = diffInputSkyfile.skycell_id
+WHERE
+    magicDSRun.state = 'run'
+    AND magicDSRun.stage = 'diff'
+    AND diffRun.state = 'full'
+    AND diffSkyfile.fault = 0
+    AND magicDSFile.component IS NULL
+) as Foo
Index: branches/bills_081204/ippTools/share/magictool_chipprocessedimfile.sql
===================================================================
--- branches/bills_081204/ippTools/share/magictool_chipprocessedimfile.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magictool_chipprocessedimfile.sql	(revision 20890)
@@ -0,0 +1,29 @@
+SELECT DISTINCT
+    chip_id,
+    class_id,
+    chipProcessedImfile.uri,
+    chipProcessedImfile.path_base,
+    camProcessedExp.path_base as cam_path_base
+FROM magicRun
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffInputSkyfile
+    ON diffInputSkyfile.diff_id = diffRun.diff_id
+    AND diffInputSkyfile.skycell_id = diffRun.skycell_id
+    -- Want input warps only
+    AND diffInputSkyfile.warp_id IS NOT NULL
+    AND diffInputSkyfile.template = 0
+JOIN warpRun USING(warp_id)
+JOIN warpSkyCellMap
+    ON warpSkyCellMap.warp_id = warpRun.warp_id
+    AND warpSkyCellMap.skycell_id = diffRun.skycell_id
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN camProcessedExp USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN chipProcessedImfile USING(chip_id,class_id)
+WHERE
+    chipRun.state = 'full'
+    AND chipProcessedImfile.fault = 0
+--   AND magicRun.state = 'stop'
+--   AND magicMask.fault = 0
Index: branches/bills_081204/ippTools/share/magictool_definebyquery.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_definebyquery.sql	(revision 20890)
@@ -21,43 +21,46 @@
 
 
+
 -- magictool_definebyquery_temp_insert.sql
 -- List of best differences for each exposure
 INSERT INTO magicBestDiffs
 SELECT
-    exp_id,
-    warpSkyfile.skycell_id,
-    warpSkyfile.tess_id,
+    rawExp.exp_id,
+    diffInputSkyfile.skycell_id,
+    diffInputSkyfile.tess_id,
     MAX(diffSkyfile.diff_id) AS diff_id
 FROM rawExp
-JOIN chipRun USING(exp_id, tess_id)
-JOIN camRun USING(chip_id, tess_id)
-JOIN fakeRun USING(cam_id, tess_id)
-JOIN warpRun USING(fake_id, tess_id)
-JOIN warpSkyCellMap USING(warp_id, tess_id)
-JOIN warpSkyfile USING(warp_id, skycell_id, tess_id)
+JOIN chipRun USING(exp_id)
+JOIN camRun USING(chip_id)
+JOIN fakeRun USING(cam_id)
+JOIN warpRun USING(fake_id)
+JOIN warpSkyCellMap USING(warp_id)
+JOIN warpSkyfile USING(warp_id, skycell_id)
 JOIN diffInputSkyfile
     ON diffInputSkyfile.warp_id = warpSkyfile.warp_id
     AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
-    AND diffInputSkyfile.tess_id = warpSkyfile.tess_id
     AND diffInputSkyfile.template = 0 -- selecting inputs only
+JOIN diffRun USING(diff_id)
 JOIN diffSkyfile USING(diff_id)
 WHERE
     diffSkyfile.fault = 0
------    AND warpSkyfile.good_frac >= 0.5
--- magictool_definebyquery_temp_insert_groupby.sql
+-- WHERE hook %s
+AND exp_id = 36475
+AND diffRun.label = 'magic_2008-11-12'
 GROUP BY
     exp_id,
-    warpSkyfile.skycell_id
+    diffInputSkyfile.skycell_id,
+    diffInputSkyfile.tess_id
 ;
 
 
--- magictool_definebyquery_select_part1.sql
--- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed
--- After this follows magictool_definebyquery_select_part2.sql
-SELECT
+-- magictool_definebyquery_select.sql
+-- Get a list of exposures on which magic may be performed
+SELECT DISTINCT
     exp_id,
     filter,
     num_todo,
-    num_done
+    num_done,
+    MAX(magic_id) AS best_magic_id
 FROM (
     -- Number of skycells as a function of exposure
@@ -65,21 +68,19 @@
         exp_id,
         filter,
-        COUNT(skycell_id) AS num_todo
+        COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo
     FROM rawExp
-    JOIN chipRun USING(exp_id, tess_id)
-    JOIN camRun USING(chip_id, tess_id)
-    JOIN fakeRun USING(cam_id, tess_id)
-    JOIN warpRun USING(fake_id, tess_id)
-    JOIN warpSkyCellMap USING(warp_id, tess_id)
-    JOIN warpSkyfile USING(warp_id, skycell_id, tess_id)
+    JOIN chipRun USING(exp_id)
+    JOIN camRun USING(chip_id)
+    JOIN fakeRun USING(cam_id)
+    JOIN warpRun USING(fake_id)
+    JOIN warpSkyCellMap USING(warp_id)
+    JOIN warpSkyfile USING(warp_id, skycell_id)
+    JOIN diffInputSkyfile USING(warp_id,skycell_id)
+    JOIN diffRun USING(diff_id)
     WHERE
         warpSkyfile.ignored = 0
-        AND warpRun.state = 'full'
-    -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac)
------        AND warpSkyfile.good_frac > 0.5
--- INSERT HERE magictool_definebyquery_select_part2.sql
--- magictool_definebyquery_select_part2.sql
--- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed
--- This follows magictool_definebyquery_select_part1.sql
+        -- magicSkycellNums WHERE hook %s
+AND exp_id = 36475
+AND diffRun.label = 'magic_2008-11-12'
     GROUP BY
         exp_id
@@ -95,7 +96,8 @@
     ) AS magicDiffNums USING(exp_id)
 LEFT JOIN magicRun USING(exp_id)
+;
+
 WHERE
-    num_done = num_todo
-    AND magicRun.magic_id IS NULL
+    magic_id IS NULL
 ;
 
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_select.sql
===================================================================
--- branches/bills_081204/ippTools/share/magictool_definebyquery_select.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magictool_definebyquery_select.sql	(revision 20890)
@@ -0,0 +1,38 @@
+-- Get a list of exposures on which magic may be performed
+SELECT DISTINCT
+    exp_id,
+    filter,
+    num_todo,
+    num_done,
+    MAX(magic_id)
+FROM (
+    -- Number of skycells as a function of exposure
+    SELECT
+        exp_id,
+        filter,
+        COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo
+    FROM rawExp
+    JOIN chipRun USING(exp_id)
+    JOIN camRun USING(chip_id)
+    JOIN fakeRun USING(cam_id)
+    JOIN warpRun USING(fake_id)
+    JOIN warpSkyCellMap USING(warp_id)
+    JOIN warpSkyfile USING(warp_id, skycell_id)
+    JOIN diffInputSkyfile USING(warp_id,skycell_id)
+    JOIN diffRun USING(diff_id)
+    WHERE
+        warpSkyfile.ignored = 0
+        -- magicSkycellNums WHERE hook %s
+    GROUP BY
+        exp_id
+    ) AS magicSkycellNums
+JOIN (
+    -- Number of completed diffs for an exposure
+    SELECT
+        exp_id,
+        COUNT(diff_id) AS num_done
+    FROM magicBestDiffs
+    GROUP BY
+        exp_id
+    ) AS magicDiffNums USING(exp_id)
+LEFT JOIN magicRun USING(exp_id)
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_select_part1.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery_select_part1.sql	(revision 20530)
+++ 	(revision )
@@ -1,25 +1,0 @@
--- This is part 1 of 2 of a query to get a list of exposures on which magic may be performed
--- After this follows magictool_definebyquery_select_part2.sql
-SELECT
-    exp_id,
-    filter,
-    num_todo,
-    num_done
-FROM (
-    -- Number of skycells as a function of exposure
-    SELECT
-        exp_id,
-        filter,
-        COUNT(DISTINCT skycell_id) AS num_todo
-    FROM rawExp
-    JOIN chipRun USING(exp_id)
-    JOIN camRun USING(chip_id)
-    JOIN fakeRun USING(cam_id)
-    JOIN warpRun USING(fake_id)
-    JOIN warpSkyCellMap USING(warp_id)
-    JOIN warpSkyfile USING(warp_id, skycell_id)
-    WHERE
-        warpSkyfile.ignored = 0
-    AND warpRun.state = 'full'
--- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac)
--- INSERT HERE magictool_definebyquery_select_part2.sql
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_select_part2.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery_select_part2.sql	(revision 20530)
+++ 	(revision )
@@ -1,18 +1,0 @@
--- This is part 2 of 2 of a query to get a list of exposures on which magic may be performed
--- This follows magictool_definebyquery_select_part1.sql
-    GROUP BY
-        exp_id
-    ) AS magicSkycellNums
-JOIN (
-    -- Number of completed diffs for an exposure
-    SELECT
-        exp_id,
-        COUNT(diff_id) AS num_done
-    FROM magicBestDiffs
-    GROUP BY
-        exp_id
-    ) AS magicDiffNums USING(exp_id)
-LEFT JOIN magicRun USING(exp_id)
-WHERE
-    num_done = num_todo
-    AND magicRun.magic_id IS NULL
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_temp_create.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery_temp_create.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_definebyquery_temp_create.sql	(revision 20890)
@@ -3,5 +3,4 @@
 skycell_id VARCHAR(64),
 tess_id VARCHAR(64),
-warp_id BIGINT,
 diff_id BIGINT,
 PRIMARY KEY(exp_id, skycell_id, tess_id)
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_temp_insert.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery_temp_insert.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_definebyquery_temp_insert.sql	(revision 20890)
@@ -3,7 +3,6 @@
 SELECT
     rawExp.exp_id,
-    warpSkyfile.skycell_id,
-    warpSkyfile.tess_id,
-    warpRun.warp_id AS warp_id,
+    diffInputSkyfile.skycell_id,
+    diffInputSkyfile.tess_id,
     MAX(diffSkyfile.diff_id) AS diff_id
 FROM rawExp
@@ -18,5 +17,12 @@
     AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
     AND diffInputSkyfile.template = 0 -- selecting inputs only
+JOIN diffRun USING(diff_id)
 JOIN diffSkyfile USING(diff_id)
 WHERE
     diffSkyfile.fault = 0
+-- WHERE hook %s
+GROUP BY
+    exp_id,
+    diffInputSkyfile.skycell_id,
+    diffInputSkyfile.tess_id
+;
Index: branches/bills_081204/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_definebyquery_temp_insert_groupby.sql	(revision 20530)
+++ 	(revision )
@@ -1,3 +1,0 @@
-GROUP BY
-    exp_id,
-    warpSkyfile.skycell_id
Index: branches/bills_081204/ippTools/share/magictool_diffskyfile.sql
===================================================================
--- branches/bills_081204/ippTools/share/magictool_diffskyfile.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magictool_diffskyfile.sql	(revision 20890)
@@ -0,0 +1,27 @@
+SELECT DISTINCT
+    diffSkyfile.diff_id,
+    diffRun.skycell_id,
+    diffSkyfile.uri,
+    diffSkyfile.path_base
+FROM magicRun
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile
+    ON diffInputSkyfile.diff_id = diffRun.diff_id
+    AND diffInputSkyfile.skycell_id = diffRun.skycell_id
+    -- Want input warps only
+    AND diffInputSkyfile.warp_id IS NOT NULL
+    AND diffInputSkyfile.template = 0
+JOIN warpSkyCellMap
+    ON warpSkyCellMap.warp_id = diffInputSkyfile.warp_id
+    AND warpSkyCellMap.skycell_id = diffInputSkyfile.skycell_id
+JOIN warpSkyfile
+    ON warpSkyfile.warp_id = warpSkyCellMap.warp_id
+    AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id
+    AND warpSkyfile.ignored = 0
+WHERE
+    diffRun.state = 'full'
+    AND diffSkyfile.fault = 0
+--   AND magicRun.state = 'stop'
+--   AND magicMask.fault = 0
Index: branches/bills_081204/ippTools/share/magictool_inputs.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_inputs.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_inputs.sql	(revision 20890)
@@ -6,4 +6,5 @@
     magicRun.state,
     magicInputSkyfile.node,
+    diffSkyfile.diff_id,
     diffSkyfile.uri,
     diffSkyfile.path_base,
@@ -20,4 +21,5 @@
     magicRun.state,
     magicTree.node,
+    0,
     magicNodeResult.uri,
     NULL, -- magicNodeResult doesn't have a path_base
Index: branches/bills_081204/ippTools/share/magictool_inputskyfile.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_inputskyfile.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_inputskyfile.sql	(revision 20890)
@@ -1,7 +1,10 @@
 SELECT
-    magicInputSkyfile.*
+    magicInputSkyfile.*,
+    diffSkyfile.uri
 FROM magicInputSkyfile
 JOIN magicRun
     USING(magic_id)
+JOIN diffSkyfile
+    USING(diff_id)
 WHERE
     magicRun.state = 'run'
Index: branches/bills_081204/ippTools/share/magictool_mask.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_mask.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_mask.sql	(revision 20890)
@@ -1,4 +1,5 @@
 SELECT
-    magicMask.*
+    magicMask.*,
+    magicRun.exp_id
 FROM magicMask
 JOIN magicRun
Index: branches/bills_081204/ippTools/share/magictool_rawimfile.sql
===================================================================
--- branches/bills_081204/ippTools/share/magictool_rawimfile.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magictool_rawimfile.sql	(revision 20890)
@@ -0,0 +1,27 @@
+SELECT DISTINCT
+    exp_id,
+    class_id,
+    rawImfile.uri,
+    camProcessedExp.path_base as cam_path_base
+FROM magicRun
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffInputSkyfile
+    ON diffInputSkyfile.diff_id = diffRun.diff_id
+    -- Want input warps only
+    AND diffInputSkyfile.warp_id IS NOT NULL
+    AND diffInputSkyfile.template = 0
+JOIN warpRun USING(warp_id)
+JOIN warpSkyCellMap
+    ON warpSkyCellMap.warp_id = warpRun.warp_id
+    AND warpSkyCellMap.skycell_id = diffRun.skycell_id
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN camProcessedExp USING(cam_id)
+JOIN chipRun USING(chip_id,exp_id)
+JOIN rawExp USING(exp_id)
+JOIN rawImfile USING(exp_id,class_id)
+WHERE
+    rawImfile.fault = 0
+--   AND magicRun.state = 'stop'
+--   AND magicMask.fault = 0
Index: branches/bills_081204/ippTools/share/magictool_toprocess_inputs.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_toprocess_inputs.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_toprocess_inputs.sql	(revision 20890)
@@ -2,5 +2,7 @@
     magicTree.*,
     rawExp.camera,
+    rawExp.exp_id,
     diffSkyfile.path_base,
+    magicRun.workdir,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
Index: branches/bills_081204/ippTools/share/magictool_toprocess_tree.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/magictool_toprocess_tree.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/magictool_toprocess_tree.sql	(revision 20890)
@@ -16,2 +16,4 @@
 WHERE
     magicRun.state = 'run'
+ORDER BY
+    magicRun.magic_id
Index: branches/bills_081204/ippTools/share/magictool_warpskyfile.sql
===================================================================
--- branches/bills_081204/ippTools/share/magictool_warpskyfile.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/magictool_warpskyfile.sql	(revision 20890)
@@ -0,0 +1,24 @@
+SELECT DISTINCT
+    warpSkyfile.warp_id,
+    warpSkyfile.skycell_id,
+    warpSkyfile.uri,
+    warpSkyfile.path_base
+FROM magicRun
+JOIN magicInputSkyfile USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffInputSkyfile
+    ON diffInputSkyfile.diff_id = diffRun.diff_id
+    AND diffInputSkyfile.skycell_id = diffRun.skycell_id
+    -- Want input warps only
+    AND diffInputSkyfile.warp_id IS NOT NULL
+    AND diffInputSkyfile.template = 0
+JOIN warpSkyfile
+    ON warpSkyfile.warp_id = diffInputSkyfile.warp_id
+    AND warpSkyfile.skycell_id = diffRun.skycell_id
+JOIN warpRun
+    ON warpRun.warp_id = warpSkyfile.warp_id
+WHERE
+    warpRun.state = 'full'
+    AND warpSkyfile.fault = 0
+--   AND magicRun.state = 'stop'
+--   AND magicMask.fault = 0
Index: branches/bills_081204/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/pxadmin_create_tables.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/pxadmin_create_tables.sql	(revision 20890)
@@ -234,4 +234,5 @@
     moon_alt FLOAT,
     moon_phase FLOAT,
+    ignored TINYINT DEFAULT 0,
     hostname VARCHAR(64),
     fault SMALLINT NOT NULL,
@@ -1058,4 +1059,32 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE magicDSRun (
+        magic_ds_id BIGINT AUTO_INCREMENT,
+        magic_id BIGINT,
+        state VARCHAR(64),
+        stage VARCHAR(64),
+        stage_id BIGINT,
+        cam_id BIGINT,
+        outroot VARCHAR(255),
+        recoveryroot VARCHAR(255),
+        re_place TINYINT,
+        remove TINYINT,
+        PRIMARY KEY(magic_ds_id),
+        KEY(magic_ds_id),
+        KEY(state),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicDSFile (
+    magic_ds_id BIGINT,
+    component VARCHAR(64),
+    backup_path_base VARCHAR(255),
+    recovery_path_base VARCHAR(255),
+    fault SMALLINT,
+    PRIMARY KEY(magic_ds_id, component),
+    KEY(fault),
+    FOREIGN KEY (magic_ds_id) REFERENCES magicDSRun(magic_ds_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE calDB (
         cal_id BIGINT AUTO_INCREMENT,
@@ -1081,7 +1110,7 @@
         det_type VARCHAR(64),
         dvodb VARCHAR(64),
-    	camera VARCHAR(64),
-    	telescope VARCHAR(64),
-	epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        camera VARCHAR(64),
+        telescope VARCHAR(64),
+        epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
         filter VARCHAR(64),
         state VARCHAR(64),
@@ -1100,5 +1129,5 @@
         corr_id BIGINT,
         chip_id BIGINT,
-	include TINYINT,
+        include TINYINT,
         PRIMARY KEY(corr_id, chip_id),
         FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
@@ -1110,5 +1139,5 @@
         chip_id BIGINT,
         cam_id BIGINT,
-	include TINYINT,
+        include TINYINT,
         PRIMARY KEY(corr_id, cam_id),
         FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
Index: branches/bills_081204/ippTools/share/pxadmin_drop_tables.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/pxadmin_drop_tables.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/pxadmin_drop_tables.sql	(revision 20890)
@@ -47,4 +47,6 @@
 DROP TABLE IF EXISTS magicMask;
 DROP TABLE IF EXISTS magicSkyfileMask;
+DROP TABLE IF EXISTS magicDSRun;
+DROP TABLE IF EXISTS magicDSFile;
 DROP TABLE IF EXISTS calDB;
 DROP TABLE IF EXISTS calRun;
Index: branches/bills_081204/ippTools/share/regtool_processedimfile.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/regtool_processedimfile.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/regtool_processedimfile.sql	(revision 20890)
@@ -7,4 +7,3 @@
 JOIN newExp
     USING(exp_id)
--- bogus conditional so there is a where clause to append to
-WHERE rawImfile.exp_id is NOT NULL
+WHERE rawImfile.ignored = 0
Index: branches/bills_081204/ippTools/share/warptool_imfile.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/warptool_imfile.sql	(revision 20530)
+++ branches/bills_081204/ippTools/share/warptool_imfile.sql	(revision 20890)
@@ -19,5 +19,6 @@
 JOIN rawImfile -- is there any reason not to refer back to rawimfiles?
     ON chipProcessedImfile.exp_id = rawImfile.exp_id
-    AND chipProcessedImfile.class_id = rawImfile.class_id 
+    AND chipProcessedImfile.class_id = rawImfile.class_id
+    AND rawImfile.ignored = 0
 WHERE
     warpRun.state = 'new'
Index: branches/bills_081204/ippTools/share/warptool_revertwarped.sql
===================================================================
--- branches/cnb_branch_20081104/ippTools/share/warptool_revertwarped.sql	(revision 20530)
+++ 	(revision )
@@ -1,9 +1,0 @@
-DELETE FROM warpSkyfile
-USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp
-WHERE
-    warpSkyfile.warp_id = warpRun.warp_id
-    AND warpRun.fake_id = fakeRun.fake_id
-    AND fakeRun.cam_id = camRun.cam_id
-    AND camRun.chip_id = chipRun.chip_id
-    AND chipRun.exp_id = rawExp.exp_id
-    AND warpSkyfile.fault != 0
Index: branches/bills_081204/ippTools/share/warptool_revertwarped_delete.sql
===================================================================
--- branches/bills_081204/ippTools/share/warptool_revertwarped_delete.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/warptool_revertwarped_delete.sql	(revision 20890)
@@ -0,0 +1,8 @@
+DELETE FROM warpSkyfile
+USING warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp
+WHERE warpSkyfile.warp_id = warpRun.warp_id
+    AND warpRun.fake_id = fakeRun.fake_id
+    AND fakeRun.cam_id = camRun.cam_id
+    AND camRun.chip_id = chipRun.chip_id
+    AND chipRun.exp_id = rawExp.exp_id
+    AND warpSkyfile.fault != 0
Index: branches/bills_081204/ippTools/share/warptool_revertwarped_update.sql
===================================================================
--- branches/bills_081204/ippTools/share/warptool_revertwarped_update.sql	(revision 20890)
+++ branches/bills_081204/ippTools/share/warptool_revertwarped_update.sql	(revision 20890)
@@ -0,0 +1,8 @@
+UPDATE warpRun
+JOIN warpSkyfile USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+SET warpRun.state = 'new'
+WHERE warpSkyfile.fault != 0
Index: branches/bills_081204/ippTools/src/.cvsignore
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/.cvsignore	(revision 20530)
+++ branches/bills_081204/ippTools/src/.cvsignore	(revision 20890)
@@ -28,4 +28,5 @@
 pxdata.c
 magictool
+magicdstool
 caltool
 flatcorr
Index: branches/bills_081204/ippTools/src/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/Makefile.am	(revision 20530)
+++ branches/bills_081204/ippTools/src/Makefile.am	(revision 20890)
@@ -8,4 +8,5 @@
 	flatcorr \
 	magictool \
+	magicdstool \
 	pstamptool \
 	pxadmin \
@@ -47,4 +48,5 @@
 	faketool.h \
 	magictool.h \
+	magicdstool.h \
 	pstamptool.h \
 	pxinject.h \
@@ -132,4 +134,10 @@
     magictoolConfig.c
 
+magicdstool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+magicdstool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+magicdstool_SOURCES = \
+    magicdstool.c \
+    magicdstoolConfig.c
+
 warptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 warptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
Index: branches/bills_081204/ippTools/src/camtool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/camtool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/camtool.c	(revision 20890)
@@ -222,4 +222,5 @@
     pxcamGetSearchArgs (config, where);
     PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "camRun.state", "==");
 
     if (!psListLength(where->list)
@@ -713,5 +714,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "==");
     pxcamGetSearchArgs (config, where);
     PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
Index: branches/bills_081204/ippTools/src/difftool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/difftool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/difftool.c	(revision 20890)
@@ -557,9 +557,10 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
     PXOPT_COPY_S16(config->args, where, "-code", "diffSkyfile.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -936,14 +937,15 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpsToDiff.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpsToDiff.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "warpsToDiff.tess_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-filter", "warpsToDiff.filter", "==");
-    PXOPT_COPY_STR(config->args, where, "-stack_label", "stacksForDiff.stack_label", "==");
-    PXOPT_COPY_STR(config->args, where, "-warp_label", "warpsToDiff.warp_label", "==");
-    PXOPT_COPY_STR(config->args, where,  "-kind", "warpsToDiff.kind", "==");
-    PXOPT_COPY_F32(config->args, where,  "-good_frac", "warpsToDiff.good_frac", ">=");
+    psMetadata *warpWhere = psMetadataAlloc();
+    psMetadata *stackWhere = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
+    PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_STR(config->args, warpWhere,  "-kind", "warpsToDiff.kind", "==");
+    PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
 
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
@@ -969,12 +971,28 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString warpQuery = NULL;
+    psString stackQuery = NULL;
+
+    if (psListLength(warpWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
+        psStringAppend(&warpQuery, "\n AND %s", whereClause);
         psFree(whereClause);
-    }
-    psFree(where);
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    } else {
+        warpQuery = psStringCopy("");
+    }
+    psFree(warpWhere);
+
+    if (psListLength(stackWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
+        psStringAppend(&stackQuery, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        stackQuery = psStringCopy("");
+    }
+    psFree(stackWhere);
+
+    psTrace("difftool", 1, query, warpQuery, stackQuery);
+
+    if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -1031,6 +1049,6 @@
         }
 
-        if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, PS_MAX_S64, PS_MAX_S64,
-                          stack_id, config)) {
+        if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id,
+                          PS_MAX_S64, PS_MAX_S64, stack_id, config)) {
             psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
                       warp_id, stack_id);
Index: branches/bills_081204/ippTools/src/difftoolConfig.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/difftoolConfig.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/difftoolConfig.c	(revision 20890)
@@ -120,4 +120,5 @@
     psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "define exposure ID", 0);
     psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name",  0,         "define exposure name", NULL);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id",  0,         "define warp_id", NULL);
     psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -156,5 +157,4 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-kind", 0, "search by kind", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", 0, "search by label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
Index: branches/bills_081204/ippTools/src/faketool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/faketool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/faketool.c	(revision 20890)
@@ -1122,8 +1122,4 @@
                     fakeRun->end_stage
         )) {
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
             psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp");
             psFree(fakeRun);
Index: branches/bills_081204/ippTools/src/magicdstool.c
===================================================================
--- branches/bills_081204/ippTools/src/magicdstool.c	(revision 20890)
+++ branches/bills_081204/ippTools/src/magicdstool.c	(revision 20890)
@@ -0,0 +1,759 @@
+/*
+ * magicdstool.c
+ *
+ * Copyright (C) 2006-2007  IfA
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVB_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+#include <ippdb.h>
+
+#include "pxtools.h"
+#include "magicdstool.h"
+
+static bool definebyqueryMode(pxConfig *config);
+static psS64 definerunMode(pxConfig *config);
+static bool updaterunMode(pxConfig *config);
+static bool todestreakMode(pxConfig *config);
+static bool adddestreakedfileMode(pxConfig *config);
+static bool revertdestreakedfileMode(pxConfig *config);
+static bool getskycellsMode(pxConfig *config);
+
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
+static bool magicDSRunComplete(pxConfig *config);
+static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id);
+
+#ifdef notdef
+static bool toprocessMode(pxConfig *config);
+static bool addresultMode(pxConfig *config);
+static bool revertnodeMode(pxConfig *config);
+static bool inputsMode(pxConfig *config);
+static bool tomaskMode(pxConfig *config);
+static bool addmaskMode(pxConfig *config);
+static bool revertmaskMode(pxConfig *config);
+static bool maskMode(pxConfig *config);
+static bool diffskyfileMode(pxConfig *config);
+static bool warpskyfileMode(pxConfig *config);
+static bool chipprocessedimfileMode(pxConfig *config);
+static bool rawimfileMode(pxConfig *config);
+
+static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
+static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);
+#endif
+
+# define MODECASE(caseName, func) \
+    case caseName: \
+    if (!func(config)) { \
+        goto FAIL; \
+    } \
+    break;
+
+int main(int argc, char **argv)
+{
+    psLibInit(NULL);
+
+    pxConfig *config = magicdstoolConfig(NULL, argc, argv);
+    if (!config) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
+        goto FAIL;
+    }
+
+    switch (config->mode) {
+        MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY,       definebyqueryMode);
+        MODECASE(MAGICDSTOOL_MODE_DEFINERUN,           definerunMode);
+        MODECASE(MAGICDSTOOL_MODE_UPDATERUN,           updaterunMode);
+        MODECASE(MAGICDSTOOL_MODE_TODESTREAK,          todestreakMode);
+        MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE,   adddestreakedfileMode);
+        MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode);
+        MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS,         getskycellsMode);
+        default:
+            psAbort("invalid option (this should not happen)");
+    }
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(EXIT_SUCCESS);
+
+FAIL:
+    psErrorStackPrint(stderr, "\n");
+    int exit_status = pxerrorGetExitStatus();
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(exit_status);
+}
+
+static bool definebyqueryMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet");
+
+    return false;
+#ifdef notyet
+
+    // Required
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+
+    // Optional
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // Create temporary table of the best diffs
+    {
+        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        psFree(query);
+    }
+
+    // Insert list of best diffs into temporary table
+    {
+        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        psMetadata *where = psMetadataAlloc();
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
+        PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+
+        psString whereClause = NULL;    // WHERE conditions
+        if (psListLength(where->list)) {
+            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringPrepend(&whereClause, "\n AND ");
+        }
+        psFree(where);
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(whereClause);
+            psFree(query);
+            return false;
+        }
+        psFree(whereClause);
+        psFree(query);
+    }
+
+    // Get list of exposures ready to magic
+    {
+        psString query = pxDataGet("magictool_definebyquery_select.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
+        {
+            psMetadata *where = psMetadataAlloc();
+            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+            PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
+            PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+
+            if (psListLength(where->list)) {
+                magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
+            }
+            psFree(where);
+        }
+
+        // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
+        // told to only take what's available.
+        // "new" means we want a new run even if there's already a magic run defined
+        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+        PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
+
+        psString queryWhere = NULL;     // WHERE conditions for entire query
+        if (available) {
+            psStringAppend(&queryWhere, " WHERE num_done = num_todo");
+        }
+        if (new) {
+            const char *newWhere = " magic_id IS NULL"; // String to add
+            if (queryWhere) {
+                psStringAppend(&queryWhere, " AND %s", newWhere);
+            } else {
+                psStringAppend(&queryWhere, " WHERE %s", newWhere);
+            }
+        }
+        if (queryWhere) {
+            psStringAppend(&query, " %s", queryWhere);
+            psFree(queryWhere);
+        }
+
+
+        if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(magicSkyCellNumsWhere);
+            psFree(query);
+            return false;
+        }
+        psFree(magicSkyCellNumsWhere);
+        psFree(query);
+    }
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+          case PS_ERR_DB_CLIENT:
+            psError(PXTOOLS_ERR_SYS, false, "database error");
+          case PS_ERR_DB_SERVER:
+            psError(PXTOOLS_ERR_PROG, false, "database error");
+          default:
+            psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // Parse the list of exposures ready to magic
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query
+
+    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i]; // Row of interest
+        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
+
+        // create a new magicRun for this group
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
+        if (!run) {
+            psAbort("failed to alloc magicRun object");
+        }
+
+        if (!magicRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            psFree(insert);
+            psFree(output);
+            psFree(list);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier
+        run->magic_id = magic_id;
+
+        psArrayAdd(list, list->n, run);
+        psFree(run);
+
+        // Create a suitable insertion query for this run
+        psString thisInsert = psStringCopy(insert);
+        {
+            psString idString = NULL;
+            psStringAppend(&idString, "%" PRId64, magic_id);
+            psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@");
+            psFree(idString);
+        }
+        {
+            psString idString = NULL;
+            psStringAppend(&idString, "%" PRId64, exp_id);
+            psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
+            psFree(idString);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id, exp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(thisInsert);
+            psFree(insert);
+            psFree(output);
+            psFree(list);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(thisInsert);
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(output);
+
+    if (!magicRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+
+    psFree(list);
+
+    return true;
+#endif // notyet
+}
+
+static psS64 definerunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+    PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false);
+    PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
+    PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false);
+    PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false);
+
+    // optional
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psS64 stage_id = 0, cam_id = 0;
+
+    if (!magicDSGetIDs(config, stage, magic_id, &stage_id, &cam_id)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to get ids");
+        return false;
+    }
+
+    magicDSRunRow *run = magicDSRunRowAlloc(
+            0,          // ID
+            magic_id,
+            "run",      // state
+            stage,
+            stage_id,
+            cam_id,
+            outroot,
+            recoveryroot,
+            re_place,
+            remove
+    );
+
+    if (!run) {
+        psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object");
+        return false;
+    }
+    if (!magicDSRunInsertObject(config->dbh, run)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(run);
+        return false;
+    }
+
+    psS64 magic_ds_id = psDBLastInsertID(config->dbh);
+    run->magic_ds_id = magic_ds_id;
+
+    if (!magicDSRunPrintObject(stdout, run, !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print object");
+            psFree(run);
+            return false;
+    }
+
+    psFree(run);
+
+    return magic_id;
+}
+
+
+static bool updaterunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+
+    if (state) {
+        // set detRun.state to state
+        return setmagicDSRunState(config, magic_ds_id, state);
+    }
+
+    return true;
+}
+
+
+static bool todestreakMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+//    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // look for "inputs" that need to processed
+    psString query = pxDataGet("magicdstool_todestreak.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool adddestreakedfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required values
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
+
+    // default values
+    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false);
+    PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+
+    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, code)) {
+            // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!magicDSRunComplete(config)) {
+            // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_PTR_NON_NULL(stage, false);
+    PS_ASSERT_PTR_NON_NULL(stage_id, false);
+    PS_ASSERT_PTR_NON_NULL(cam_id, false);
+
+    if (!strcmp(stage, "diff")) {
+        // don't need these ids for diff stage
+        *stage_id = 0;
+        *cam_id = 0;
+        return true;
+    }
+
+    int stageNum;
+    if (!strcmp(stage, "raw")) {
+        stageNum = 0;
+    } else if (!strcmp(stage, "chip")) {
+        stageNum = 1;
+    } else if (!strcmp(stage, "warp")) {
+        stageNum = 2;
+    } else {
+        psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage);
+        return false;
+    }
+
+
+    psString query = pxDataGet("magicdstool_getrunids.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query, magic_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (psArrayLength(output) > 1) {
+        psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_id %" PRId64,
+            psArrayLength(output), magic_id);
+        return false;
+    }
+    psMetadata *row = output->data[0];
+
+    *cam_id = psMetadataLookupS64(NULL, row, "cam_id");
+    if (stageNum == 0) {
+        *stage_id = psMetadataLookupS64(NULL, row, "exp_id");
+    } else if (stageNum == 1) {
+        *stage_id = psMetadataLookupS64(NULL, row, "chip_id");
+    } else if (stageNum == 2) {
+        *stage_id = psMetadataLookupS64(NULL, row, "warp_id");
+    }
+
+    return true;
+}
+
+static bool magicDSRunComplete(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // look for completed magicDSRuns
+    psString query = pxDataGet("magicdstool_completed_runs.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
+
+        // set magicDSRun.state to 'stop'
+        if (!setmagicDSRunState(config, magic_ds_id, "stop")) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
+                magic_ds_id);
+            psFree(output);
+            return false;
+        }
+    }
+
+
+    return true;
+}
+
+
+static bool revertdestreakedfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
+    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
+
+    psString query = psStringCopy("DELETE FROM magicDSFile WHERE fault != 0");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to revert");
+        return false;
+    }
+    return true;
+}
+
+static bool getskycellsMode(pxConfig *config)
+{
+    // required
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magicdstool_getskycells.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query, magic_ds_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    // check that state is a valid string value
+    if (!(
+            (strncmp(state, "run", 4) == 0)
+            || (strncmp(state, "stop", 5) == 0)
+        )
+    ) {
+        psError(PS_ERR_UNKNOWN, false,
+                "invalid magicDSRun state: %s", state);
+        return false;
+    }
+
+    char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
+    if (!p_psDBRunQuery(config->dbh, query, state, magic_ds_id)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for magic_id %" PRId64, magic_ds_id);
+        return false;
+    }
+
+    return true;
+}
+
Index: branches/bills_081204/ippTools/src/magicdstool.h
===================================================================
--- branches/bills_081204/ippTools/src/magicdstool.h	(revision 20890)
+++ branches/bills_081204/ippTools/src/magicdstool.h	(revision 20890)
@@ -0,0 +1,38 @@
+/*
+ * magicdstool.h
+ *
+ * Copyright (C) 2007  IfA
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef MAGICDSTOOL_H
+#define MAGICDSTOOL_H 1
+
+#include "pxtools.h"
+
+typedef enum {
+    MAGICDSTOOL_MODE_NONE           = 0x0,
+    MAGICDSTOOL_MODE_DEFINEBYQUERY,
+    MAGICDSTOOL_MODE_DEFINERUN,
+    MAGICDSTOOL_MODE_UPDATERUN,
+    MAGICDSTOOL_MODE_TODESTREAK,
+    MAGICDSTOOL_MODE_ADDDESTREAKEDFILE,
+    MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,
+    MAGICDSTOOL_MODE_GETSKYCELLS,
+} MAGICDStoolMode;
+
+pxConfig *magicdstoolConfig(pxConfig *config, int argc, char **argv);
+
+#endif // MAGICDSTOOL_H
Index: branches/bills_081204/ippTools/src/magicdstoolConfig.c
===================================================================
--- branches/bills_081204/ippTools/src/magicdstoolConfig.c	(revision 20890)
+++ branches/bills_081204/ippTools/src/magicdstoolConfig.c	(revision 20890)
@@ -0,0 +1,157 @@
+/*
+ * magicdstoolConfig.c
+ *
+ * Copyright (C) 2007  IfA
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdint.h>
+
+#include <psmodules.h>
+
+#include "pxtools.h"
+#include "magicdstool.h"
+
+pxConfig *magicdstoolConfig(pxConfig *config, int argc, char **argv)
+{
+    if (!config) {
+        config = pxConfigAlloc();
+    }
+
+    pmConfigReadParamsSet(false);
+
+    // setup site config
+    config->modules = pmConfigRead(&argc, argv, NULL);
+    if (!config->modules) {
+        psError(PS_ERR_UNKNOWN, false, "Can't find site configuration");
+        psFree(config);
+        return NULL;
+    }
+
+    psTime *now = psTimeGetNow(PS_TIME_TAI);
+
+    // -definebyquery
+    psMetadata *definebyqueryArgs = psMetadataAlloc();
+#ifdef notyet
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",     0, "define workdir (required)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",       0, "define label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
+    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+#endif
+
+    // -definerun
+    psMetadata *definerunArgs = psMetadataAlloc();
+    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0);
+    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL);
+    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL);
+    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false);
+//    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -updaterun
+    psMetadata *updaterunArgs = psMetadataAlloc();
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL);
+
+    // -addinputskyfile
+    psMetadata *addinputskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0);
+    psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
+
+    // -todestreak
+    psMetadata *todestreakArgs = psMetadataAlloc();
+    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
+    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
+//    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -adddestreakedfile
+    psMetadata *adddestreakedfileArgs = psMetadataAlloc();
+    psMetadataAddS64(adddestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "define component name (required)", NULL);
+    psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-backup_path_base", 0, "define backup URI", NULL);
+    psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL);
+    psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0);
+
+    // -revertdestreakedfile
+    psMetadata *revertdestreakedfileArgs = psMetadataAlloc();
+    psMetadataAddS64(revertdestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);
+    psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL);
+    psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0);
+
+    // -getskycells
+    psMetadata *getskycellsArgs = psMetadataAlloc();
+    psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0);
+    psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL);
+    psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL);
+    psMetadataAddBool(getskycellsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    psFree(now);
+
+    psMetadata *argSets = psMetadataAlloc();
+    psMetadata *modes   = psMetadataAlloc();
+
+    PXOPT_ADD_MODE("-definebyquery",       "create magic de-streak runs from magic runs",
+                    MAGICDSTOOL_MODE_DEFINEBYQUERY,     definebyqueryArgs);
+    PXOPT_ADD_MODE("-definerun",           "create one magic de-streak run for one magic run",
+                    MAGICDSTOOL_MODE_DEFINERUN,         definerunArgs);
+    PXOPT_ADD_MODE("-updaterun",           "update state of magic de-streak run",
+                    MAGICDSTOOL_MODE_UPDATERUN,         updaterunArgs);
+    PXOPT_ADD_MODE("-todestreak",          "show pending files",    
+                    MAGICDSTOOL_MODE_TODESTREAK,       todestreakArgs);
+    PXOPT_ADD_MODE("-adddestreakedfile",   "add a de-streaked file",
+                    MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs);
+    PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file",
+                    MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs);
+    PXOPT_ADD_MODE("-getskycells", "get skycell files ",
+                    MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsArgs);
+
+    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
+        psError(PS_ERR_UNKNOWN, true, "option parsing failed");
+        psFree(argSets);
+        psFree(modes);
+        psFree(config);
+        return NULL;
+    }
+
+    psFree(argSets);
+    psFree(modes);
+
+    // define Database handle, if used
+    // do this last so we don't setup a connection before CLI options are
+    // validated
+    config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules));
+    if (!config->dbh) {
+        psError(PS_ERR_UNKNOWN, false, "Can't configure database");
+        psFree(config);
+        return NULL;
+    }
+
+    return config;
+}
Index: branches/bills_081204/ippTools/src/magictool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/magictool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/magictool.c	(revision 20890)
@@ -47,4 +47,9 @@
 static bool revertmaskMode(pxConfig *config);
 static bool maskMode(pxConfig *config);
+static bool diffskyfileMode(pxConfig *config);
+static bool warpskyfileMode(pxConfig *config);
+static bool chipprocessedimfileMode(pxConfig *config);
+static bool rawimfileMode(pxConfig *config);
+
 
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
@@ -69,20 +74,24 @@
 
     switch (config->mode) {
-        MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY,  definebyqueryMode);
-        MODECASE(MAGICTOOL_MODE_DEFINERUN,      definerunMode);
-        MODECASE(MAGICTOOL_MODE_UPDATERUN,      updaterunMode);
-        MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode);
-        MODECASE(MAGICTOOL_MODE_INPUTSKYFILE,   inputskyfileMode);
-        MODECASE(MAGICTOOL_MODE_TOTREE,         totreeMode);
-        MODECASE(MAGICTOOL_MODE_INPUTTREE,      inputtreeMode);
-        MODECASE(MAGICTOOL_MODE_REVERTTREE,     reverttreeMode);
-        MODECASE(MAGICTOOL_MODE_TOPROCESS,      toprocessMode);
-        MODECASE(MAGICTOOL_MODE_ADDRESULT,      addresultMode);
-        MODECASE(MAGICTOOL_MODE_REVERTNODE,     revertnodeMode);
-        MODECASE(MAGICTOOL_MODE_INPUTS,         inputsMode);
-        MODECASE(MAGICTOOL_MODE_TOMASK,         tomaskMode);
-        MODECASE(MAGICTOOL_MODE_ADDMASK,        addmaskMode);
-        MODECASE(MAGICTOOL_MODE_REVERTMASK,     revertmaskMode);
-        MODECASE(MAGICTOOL_MODE_MASK,           maskMode);
+        MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY,       definebyqueryMode);
+        MODECASE(MAGICTOOL_MODE_DEFINERUN,           definerunMode);
+        MODECASE(MAGICTOOL_MODE_UPDATERUN,           updaterunMode);
+        MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,     addinputskyfileMode);
+        MODECASE(MAGICTOOL_MODE_INPUTSKYFILE,        inputskyfileMode);
+        MODECASE(MAGICTOOL_MODE_TOTREE,              totreeMode);
+        MODECASE(MAGICTOOL_MODE_INPUTTREE,           inputtreeMode);
+        MODECASE(MAGICTOOL_MODE_REVERTTREE,          reverttreeMode);
+        MODECASE(MAGICTOOL_MODE_TOPROCESS,           toprocessMode);
+        MODECASE(MAGICTOOL_MODE_ADDRESULT,           addresultMode);
+        MODECASE(MAGICTOOL_MODE_REVERTNODE,          revertnodeMode);
+        MODECASE(MAGICTOOL_MODE_INPUTS,              inputsMode);
+        MODECASE(MAGICTOOL_MODE_TOMASK,              tomaskMode);
+        MODECASE(MAGICTOOL_MODE_ADDMASK,             addmaskMode);
+        MODECASE(MAGICTOOL_MODE_REVERTMASK,          revertmaskMode);
+        MODECASE(MAGICTOOL_MODE_MASK,                maskMode);
+        MODECASE(MAGICTOOL_MODE_DIFFSKYFILE,         diffskyfileMode);
+        MODECASE(MAGICTOOL_MODE_WARPSKYFILE,         warpskyfileMode);
+        MODECASE(MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileMode);
+        MODECASE(MAGICTOOL_MODE_RAWIMFILE,           rawimfileMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -144,28 +153,21 @@
         psMetadata *where = psMetadataAlloc();
         PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
         PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
 
+        psString whereClause = NULL;    // WHERE conditions
         if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
+            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringPrepend(&whereClause, "\n AND ");
+        }
+        psFree(where);
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(whereClause);
-        }
-        psFree(where);
-
-        psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql");
-        if (!groupby) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             psFree(query);
             return false;
         }
-
-        psStringAppend(&query, " %s", groupby);
-        psFree(groupby);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            return false;
-        }
+        psFree(whereClause);
         psFree(query);
     }
@@ -173,5 +175,5 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select_part1.sql");
+        psString query = pxDataGet("magictool_definebyquery_select.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -179,33 +181,52 @@
         }
 
+        psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
         {
             psMetadata *where = psMetadataAlloc();
             PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+            PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
             PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
 
             if (psListLength(where->list)) {
-                psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-                psStringAppend(&query, " AND %s", whereClause);
-                psFree(whereClause);
+                magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
             }
             psFree(where);
         }
 
-        psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql");
-        if (!part2) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psStringAppend(&query, " %s", part2);
-        psFree(part2);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
+        // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
+        // told to only take what's available.
+        // "new" means we want a new run even if there's already a magic run defined
+        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+        PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
+
+        psString queryWhere = NULL;     // WHERE conditions for entire query
+        if (available) {
+            psStringAppend(&queryWhere, " WHERE num_done = num_todo");
+        }
+        if (new) {
+            const char *newWhere = " magic_id IS NULL"; // String to add
+            if (queryWhere) {
+                psStringAppend(&queryWhere, " AND %s", newWhere);
+            } else {
+                psStringAppend(&queryWhere, " WHERE %s", newWhere);
+            }
+        }
+        if (queryWhere) {
+            psStringAppend(&query, " %s", queryWhere);
+            psFree(queryWhere);
+        }
+
+
+        if (!p_psDBRunQuery(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
             psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(magicSkyCellNumsWhere);
             psFree(query);
             return false;
         }
-        psFree(query);
-    }
+        psFree(magicSkyCellNumsWhere);
+        psFree(query);
+    }
+
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
@@ -227,4 +248,6 @@
         return true;
     }
+
+    // Parse the list of exposures ready to magic
 
     if (!psDBTransaction(config->dbh)) {
@@ -346,5 +369,5 @@
     }
 
-    // get the assigned warp_id
+    // get the assigned magic_id
     psS64 magic_id = psDBLastInsertID(config->dbh);
     run->magic_id = magic_id;
@@ -801,6 +824,4 @@
     if (!psArrayLength(output)) {
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        // psFree(output);
-        // return true;
     }
 
@@ -843,33 +864,59 @@
     }
 
-    psHash *forest = psHashAlloc(psArrayLength(magicTree));
-
-    // convert the array of metadata into a pxTree structure
-    for (long i = 0; i < psArrayLength(magicTree); i++) {
+    // entries are ordered by magic_id
+    long index = 0;
+    while (index <  psArrayLength(magicTree)) {
         bool status;
-        psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
+        psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id");
         if (!status) {
-            psAbort("failed to lookup value for node column");
-        }
-
-        psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
-        if (!status) {
-            psAbort("failed to lookup value for dep column");
-        }
-
-        pxTreeBuilder(forest, node, dep, magicTree->data[i]);
-
-    }
-    psFree(magicTree);
-
-    // find the root of the tree
-    pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
-    psFree(forest);
-    //    pxTreePrint(stdout, root);
-
-    // crawl through the tree and looking for nodes with children that are all
-    // "done"
-    pxTreeCrawl(root, findReadyNodes, output);
-    psFree(root);
+            psAbort("failed to lookup value for magic_id column");
+        }
+
+        // find the end of this block
+        long first = index;
+        long last = index;
+        for (long i = index + 1; i < psArrayLength(magicTree); i++) {
+            psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id");
+            if (!status) {
+                psAbort("failed to lookup value for magic_id column");
+            }
+            if (magic_id != current_magic_id) {
+                break;
+            }
+            last = i;
+        }
+
+        index = last + 1;
+
+        psHash *forest = psHashAlloc(last - first + 1);
+
+        // convert the array of metadata into a pxTree structure
+        for (long i = first; i <= last; i++) {
+            bool status;
+            psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
+            if (!status) {
+                psAbort("failed to lookup value for node column");
+            }
+
+            psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
+            if (!status) {
+                psAbort("failed to lookup value for dep column");
+            }
+
+            pxTreeBuilder(forest, node, dep, magicTree->data[i]);
+
+        }
+
+        // find the root of the tree
+        pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
+        psFree(forest);
+        //    pxTreePrint(stdout, root);
+
+        // crawl through the tree and looking for nodes with children that are all
+        // "done"
+        pxTreeCrawl(root, findReadyNodes, output);
+        psFree(root);
+
+    }
 
     if (psArrayLength(output)) {
@@ -882,4 +929,5 @@
     }
 
+    psFree(magicTree);
     psFree(output);
     psFree(whereClause);
@@ -1207,4 +1255,265 @@
 
 
+static bool diffskyfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "==");
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_diffskyfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool warpskyfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_warpskyfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicWarpSkyfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool chipprocessedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "==");
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_chipprocessedimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicChipProcessedImfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool rawimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "==");
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_rawimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicRawImfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
 {
Index: branches/bills_081204/ippTools/src/magictool.h
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/magictool.h	(revision 20530)
+++ branches/bills_081204/ippTools/src/magictool.h	(revision 20890)
@@ -41,4 +41,8 @@
     MAGICTOOL_MODE_REVERTMASK,
     MAGICTOOL_MODE_MASK,
+    MAGICTOOL_MODE_DIFFSKYFILE,
+    MAGICTOOL_MODE_WARPSKYFILE,
+    MAGICTOOL_MODE_CHIPPROCESSEDIMFILE,
+    MAGICTOOL_MODE_RAWIMFILE,
 } MAGICtoolMode;
 
Index: branches/bills_081204/ippTools/src/magictoolConfig.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/magictoolConfig.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/magictoolConfig.c	(revision 20890)
@@ -54,5 +54,8 @@
     psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
@@ -151,4 +154,29 @@
     psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
+    // -diffskyfile
+    psMetadata *diffskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -warpskyfile
+    psMetadata *warpskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(warpskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddBool(warpskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddStr(warpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL);
+
+    // -chipprocessedimfile
+    psMetadata *chipprocessedimfileArgs = psMetadataAlloc();
+    psMetadataAddS64(chipprocessedimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(chipprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL);
+    psMetadataAddBool(chipprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -rawimfile
+    psMetadata *rawimfileArgs = psMetadataAlloc();
+    psMetadataAddS64(rawimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(rawimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL);
+    psMetadataAddBool(rawimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
     psFree(now);
 
@@ -156,20 +184,24 @@
     psMetadata *modes   = psMetadataAlloc();
 
-    PXOPT_ADD_MODE("-definebyquery",   "", MAGICTOOL_MODE_DEFINEBYQUERY,   definebyqueryArgs);
-    PXOPT_ADD_MODE("-definerun",       "", MAGICTOOL_MODE_DEFINERUN,       definerunArgs);
-    PXOPT_ADD_MODE("-updaterun",       "", MAGICTOOL_MODE_UPDATERUN,       updaterunArgs);
-    PXOPT_ADD_MODE("-addinputskyfile", "", MAGICTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs);
-    PXOPT_ADD_MODE("-inputskyfile",    "", MAGICTOOL_MODE_INPUTSKYFILE,    inputskyfileArgs);
-    PXOPT_ADD_MODE("-totree",          "", MAGICTOOL_MODE_TOTREE,          totreeArgs);
-    PXOPT_ADD_MODE("-inputtree",       "", MAGICTOOL_MODE_INPUTTREE,       inputtreeArgs);
-    PXOPT_ADD_MODE("-reverttree",      "", MAGICTOOL_MODE_REVERTTREE,      reverttreeArgs);
-    PXOPT_ADD_MODE("-toprocess",       "", MAGICTOOL_MODE_TOPROCESS,       toprocessArgs);
-    PXOPT_ADD_MODE("-inputs",          "", MAGICTOOL_MODE_INPUTS,          inputsArgs);
-    PXOPT_ADD_MODE("-addresult",       "", MAGICTOOL_MODE_ADDRESULT,       addresultArgs);
-    PXOPT_ADD_MODE("-revertnode",      "", MAGICTOOL_MODE_REVERTNODE,      revertnodeArgs);
-    PXOPT_ADD_MODE("-tomask",          "", MAGICTOOL_MODE_TOMASK,          tomaskArgs);
-    PXOPT_ADD_MODE("-addmask",         "", MAGICTOOL_MODE_ADDMASK,         addmaskArgs);
-    PXOPT_ADD_MODE("-revertmask",      "", MAGICTOOL_MODE_REVERTMASK,      revertmaskArgs);
-    PXOPT_ADD_MODE("-mask",            "", MAGICTOOL_MODE_MASK,            maskArgs);
+    PXOPT_ADD_MODE("-definebyquery",       "", MAGICTOOL_MODE_DEFINEBYQUERY,       definebyqueryArgs);
+    PXOPT_ADD_MODE("-definerun",           "", MAGICTOOL_MODE_DEFINERUN,           definerunArgs);
+    PXOPT_ADD_MODE("-updaterun",           "", MAGICTOOL_MODE_UPDATERUN,           updaterunArgs);
+    PXOPT_ADD_MODE("-addinputskyfile",     "", MAGICTOOL_MODE_ADDINPUTSKYFILE,     addinputskyfileArgs);
+    PXOPT_ADD_MODE("-inputskyfile",        "", MAGICTOOL_MODE_INPUTSKYFILE,        inputskyfileArgs);
+    PXOPT_ADD_MODE("-totree",              "", MAGICTOOL_MODE_TOTREE,              totreeArgs);
+    PXOPT_ADD_MODE("-inputtree",           "", MAGICTOOL_MODE_INPUTTREE,           inputtreeArgs);
+    PXOPT_ADD_MODE("-reverttree",          "", MAGICTOOL_MODE_REVERTTREE,          reverttreeArgs);
+    PXOPT_ADD_MODE("-toprocess",           "", MAGICTOOL_MODE_TOPROCESS,           toprocessArgs);
+    PXOPT_ADD_MODE("-inputs",              "", MAGICTOOL_MODE_INPUTS,              inputsArgs);
+    PXOPT_ADD_MODE("-addresult",           "", MAGICTOOL_MODE_ADDRESULT,           addresultArgs);
+    PXOPT_ADD_MODE("-revertnode",          "", MAGICTOOL_MODE_REVERTNODE,          revertnodeArgs);
+    PXOPT_ADD_MODE("-tomask",              "", MAGICTOOL_MODE_TOMASK,              tomaskArgs);
+    PXOPT_ADD_MODE("-addmask",             "", MAGICTOOL_MODE_ADDMASK,             addmaskArgs);
+    PXOPT_ADD_MODE("-revertmask",          "", MAGICTOOL_MODE_REVERTMASK,          revertmaskArgs);
+    PXOPT_ADD_MODE("-mask",                "", MAGICTOOL_MODE_MASK,                maskArgs);
+    PXOPT_ADD_MODE("-diffskyfile",         "", MAGICTOOL_MODE_DIFFSKYFILE,         diffskyfileArgs);
+    PXOPT_ADD_MODE("-warpskyfile",         "", MAGICTOOL_MODE_WARPSKYFILE,         warpskyfileArgs);
+    PXOPT_ADD_MODE("-chipprocessedimfile", "", MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileArgs);
+    PXOPT_ADD_MODE("-rawimfile",           "", MAGICTOOL_MODE_RAWIMFILE,           rawimfileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
Index: branches/bills_081204/ippTools/src/pxcam.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/pxcam.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/pxcam.c	(revision 20890)
@@ -156,5 +156,5 @@
     }
 
-    psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'");
+    psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.state = '%s'");
 
     if (where) {
Index: branches/bills_081204/ippTools/src/pxchip.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/pxchip.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/pxchip.c	(revision 20890)
@@ -72,4 +72,5 @@
     psMetadataAddF32(md,  PS_LIST_TAIL, "-solang_max",         0, "search by max solar angle", NAN);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-comment",            0, "search by comment field (LIKE comparison)", NULL);
+    psMetadataAddStr(md,  PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
     return true;
 }
@@ -123,4 +124,5 @@
     PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<");
     PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "==");
     return true;
 }
Index: branches/bills_081204/ippTools/src/pzgetimfiles.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/pzgetimfiles.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/pzgetimfiles.c	(revision 20890)
@@ -114,5 +114,10 @@
     psArray *newImfiles = parseFiles(config, cmdOutput);
     if (!newImfiles) {
-        // XXX not nessicarily an error
+        // XXX not nessicarily an error but we don't want to keep trying to
+        // download an "empty" fileset.
+        // mark the summitExp row as faulted
+        if (!p_psDBRunQuery(config->dbh, "UPDATE summitExp SET fault = %d WHERE exp_name = '%s' AND camera = '%s' AND telescope = '%s'", 250, filesetid, camera, telescope)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         psError(PS_ERR_UNKNOWN, true, "no new files/imfiles");
         psFree(cmdOutput);
Index: branches/bills_081204/ippTools/src/regtool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/regtool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/regtool.c	(revision 20890)
@@ -231,4 +231,5 @@
     PXOPT_LOOKUP_F32(moon_alt,   config->args, "-moon_alt",   false, false);
     PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false);
+    PXOPT_LOOKUP_BOOL(ignored, config->args, "-ignore", false);
     PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
@@ -290,9 +291,10 @@
         user_5,
         object,
-	sun_angle,
-	sun_alt,
-	moon_angle,
-	moon_alt,
-	moon_phase,
+        sun_angle,
+        sun_alt,
+        moon_angle,
+        moon_alt,
+        moon_phase,
+        ignored,
         hostname,
         code,
@@ -404,5 +406,5 @@
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-	psFree(where);
+        psFree(where);
         return false;
     }
@@ -443,5 +445,5 @@
     if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-	psFree (where);
+        psFree (where);
         return false;
     }
@@ -750,9 +752,9 @@
         user_5,
         object,
-	sun_angle,
-	sun_alt,
-	moon_angle,
-	moon_alt,
-	moon_phase,
+        sun_angle,
+        sun_alt,
+        moon_angle,
+        moon_alt,
+        moon_phase,
         hostname,
         code,
@@ -954,5 +956,5 @@
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-	psFree(where);
+        psFree(where);
         return false;
     }
@@ -961,5 +963,5 @@
         psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp");
         psStringAppend(&query, " AND %s", whereClause);
-	psFree(where);
+        psFree(where);
         psFree(whereClause);
     }
@@ -993,5 +995,5 @@
     if (!pxSetFaultCode(config->dbh, "rawExp", where, code)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-	psFree(where);
+        psFree(where);
         return false;
     }
Index: branches/bills_081204/ippTools/src/regtoolConfig.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/regtoolConfig.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/regtoolConfig.c	(revision 20890)
@@ -119,4 +119,5 @@
     ADD_OPT(F32,  addprocessedimfileArgs, "-moon_alt",       "define moon altitude (neg = below horizon)", NAN);
     ADD_OPT(F32,  addprocessedimfileArgs, "-moon_phase",     "define moon phase (0.0 = new)",   NAN);
+    ADD_OPT(Bool, addprocessedimfileArgs, "-ignore",         "ignore this imfile?", false);
     ADD_OPT(Time, addprocessedimfileArgs, "-dateobs",        "define observation time",         NULL);
     ADD_OPT(Str,  addprocessedimfileArgs, "-hostname",       "define host name",                NULL);
@@ -159,6 +160,6 @@
     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-filelevel",        0,        "define the data partitioning level of this file (required)", NULL);
 
-    ADD_OPT(F32,  addprocessedexpArgs, "-longitude", 	    "specify the observatory longitude (NOTE: not saved in db)", 0.0);
-    ADD_OPT(F32,  addprocessedexpArgs, "-latitude",  	    "specify the observatory latitude (NOTE: not saved in db)", 0.0);
+    ADD_OPT(F32,  addprocessedexpArgs, "-longitude",        "specify the observatory longitude (NOTE: not saved in db)", 0.0);
+    ADD_OPT(F32,  addprocessedexpArgs, "-latitude",         "specify the observatory latitude (NOTE: not saved in db)", 0.0);
 
     ADD_OPT(Time, addprocessedexpArgs, "-dateobs",          "define observation time", NULL);
Index: branches/bills_081204/ippTools/src/warptool.c
===================================================================
--- branches/cnb_branch_20081104/ippTools/src/warptool.c	(revision 20530)
+++ branches/bills_081204/ippTools/src/warptool.c	(revision 20890)
@@ -1213,25 +1213,91 @@
     }
 
-    psString query = pxDataGet("warptool_revertwarped.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(where);
         return false;
     }
 
-    if (where && psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
+
+    // Update state to 'new'
+    int numUpdated;                     // Number updated
+    {
+        psString query = pxDataGet("warptool_revertwarped_update.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&query, " AND %s", whereClause);
+            psFree(whereClause);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(query);
+
+        numUpdated = psDBAffectedRows(config->dbh);
+
+        if (numUpdated < 1) {
+            psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+    }
+
+    psLogMsg("warptool", PS_LOG_INFO, "Updated %d warp runs", numUpdated);
+
+    // Delete product
+    int numDeleted;                     // Number deleted
+    {
+        psString query = pxDataGet("warptool_revertwarped_delete.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&query, " AND %s", whereClause);
+            psFree(whereClause);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(query);
+
+        numDeleted = psDBAffectedRows(config->dbh);
+    }
+
+    psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted);
 
     psFree(where);
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
 
     return true;
Index: branches/bills_081204/ippconfig/dvo.photcodes
===================================================================
--- branches/cnb_branch_20081104/ippconfig/dvo.photcodes	(revision 20530)
+++ branches/bills_081204/ippconfig/dvo.photcodes	(revision 20890)
@@ -501,10 +501,17 @@
   3600  ISP-Apogee-01.y      dep  14.150  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
 													       																		
-  4100  SIMTEST.g.00         dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  4200  SIMTEST.r.00         dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  4300  SIMTEST.i.00         dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  4400  SIMTEST.z.00         dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  4500  SIMTEST.y.00         dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+  4100  SIMTEST.g.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  4200  SIMTEST.r.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  4300  SIMTEST.i.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  4400  SIMTEST.z.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  4500  SIMTEST.y.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+
+ 14100  SIMTEST.g.SkyChip    dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+ 14200  SIMTEST.r.SkyChip    dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+ 14300  SIMTEST.i.SkyChip    dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+ 14400  SIMTEST.z.SkyChip    dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+ 14500  SIMTEST.y.SkyChip    dep  25.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+
+
   4101  SIMMOSAIC.g.00       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   4102  SIMMOSAIC.g.01       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -838,4 +845,10 @@
   10476 GPC1.y.XY76          dep  23.600  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
 
+  11000 GPC1.g.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  11100 GPC1.r.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  11200 GPC1.i.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  11300 GPC1.z.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  11400 GPC1.y.SkyChip       dep  25.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+
 # mosaic2 photcodes
 
Index: branches/bills_081204/ippconfig/gpc1/camera.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/gpc1/camera.config	(revision 20530)
+++ branches/bills_081204/ippconfig/gpc1/camera.config	(revision 20890)
@@ -92,4 +92,7 @@
         CHIP    STR     {CHIP.NAME}
         CELL    STR     {CHIP.NAME}:{CELL.NAME}
+        fpa     STR     fpa
+        chip    STR     {CHIP.NAME}
+        cell    STR     {CHIP.NAME}:{CELL.NAME}
 END
 
Index: branches/bills_081204/ippconfig/gpc1/dvo.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/gpc1/dvo.config	(revision 20530)
+++ branches/bills_081204/ippconfig/gpc1/dvo.config	(revision 20890)
@@ -67,3 +67,3 @@
 PM_TOOFEW               4
 POS_TOOFEW              1
-
+RELASTRO_SIGMA_LIM      0.01  # select measurements with dM < SIGMA_LIM for relphot analysis
Index: branches/bills_081204/ippconfig/gpc1/ppImage.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/gpc1/ppImage.config	(revision 20530)
+++ branches/bills_081204/ippconfig/gpc1/ppImage.config	(revision 20890)
@@ -39,14 +39,40 @@
 
 # Standard chip processing
-CHIP	METADATA
-  BASE.FITS        BOOL    FALSE           # Save base detrended image?
-  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
-  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
-  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
-  CHIP.MASK.FITS   BOOL    TRUE            # Save chip-mosaic-ed image? 
-  CHIP.WEIGHT.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
-  OVERSCAN         BOOL    TRUE            # Overscan subtraction
-  BIAS             BOOL    FALSE           # Bias subtraction
-  DARK             BOOL    TRUE            # Dark subtraction
+CHIP               METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.WEIGHT.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  REMNANCE         BOOL    TRUE            # Remnance masking
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    TRUE            # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM           BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BACKGROUND       BOOL    TRUE            # Subtract background?
+END
+
+# Standard chip processing for areas with high stellar density
+# Turn remnance masking off --- it kills all the detector
+CHIP_DENSE_STARS	METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.WEIGHT.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  REMNANCE         BOOL    FALSE           # Remnance masking
   SHUTTER          BOOL    FALSE           # Shutter correction
   FLAT             BOOL    TRUE            # Flat-field normalisation
@@ -265,8 +291,66 @@
 
   DETREND.CONSTRAINTS  METADATA
+    BIAS METADATA
+    END
+    DARK METADATA
+      EXPTIME STR FPA.EXPOSURE
+    END
     FLAT METADATA
       DETTYPE STR FLAT_RAW
-    END
-  END   
+      FILTER  STR FPA.FILTERID
+    END
+    FRINGE METADATA
+      FILTER   STR FPA.FILTER
+    END
+    SHUTTER METADATA
+    END
+    MASK METADATA
+    END	
+    ASTROM METADATA
+    END	
+  END
+END
+
+# Overscan, bias, dark, shutter, flat-field, fringe, photom
+PPIMAGE_FLATTEST    METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.WEIGHT.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    TRUE            # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM           BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+
+  DETREND.CONSTRAINTS  METADATA
+    BIAS METADATA
+    END
+    DARK METADATA
+      EXPTIME STR FPA.EXPOSURE
+    END
+    FLAT METADATA
+      DETTYPE STR FLATTEST
+      FILTER  STR FPA.FILTERID
+    END
+    FRINGE METADATA
+      FILTER   STR FPA.FILTER
+    END
+    SHUTTER METADATA
+    END
+    MASK METADATA
+    END	
+    ASTROM METADATA
+    END	
+  END
 END
 
Index: branches/bills_081204/ippconfig/gpc1/psastro.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/gpc1/psastro.config	(revision 20530)
+++ branches/bills_081204/ippconfig/gpc1/psastro.config	(revision 20890)
@@ -63,5 +63,5 @@
 
 # PSASTRO.FIX.CHIPS             BOOL     TRUE
-PSASTRO.PIXEL.TOLERANCE       F32      20.0
+PSASTRO.PIXEL.TOLERANCE       F32      3.0
 PSASTRO.ANGLE.TOLERANCE       F32      1.0
 
@@ -73,5 +73,5 @@
 # PSASTRO.MOSAIC.MAX.ERROR.N3 F32    0.50 # max allow error for valid solution (arcsec)
 
-PSASTRO.MOSAIC.MAX.ERROR.N0 F32    5.00 # max allow error for valid solution (arcsec)
+PSASTRO.MOSAIC.MAX.ERROR.N0 F32    0.00 # max allow error for valid solution (arcsec)
 PSASTRO.MOSAIC.MAX.ERROR.N1 F32    0.00 # max allow error for valid solution (arcsec)
 PSASTRO.MOSAIC.MAX.ERROR.N2 F32    0.00 # max allow error for valid solution (arcsec)
@@ -137,2 +137,4 @@
   PHOTCODE STR y
 END
+
+REFSTAR_MASK                    BOOL TRUE
Index: branches/bills_081204/ippconfig/gpc1/psphot.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/gpc1/psphot.config	(revision 20530)
+++ branches/bills_081204/ippconfig/gpc1/psphot.config	(revision 20890)
@@ -30,10 +30,15 @@
 PSF_CLUMP_GRID_SCALE F32 2.5		# size of Sx,Sy image pixel
 
+PSF.RESIDUALS       BOOL  TRUE            # generate the residuals?
+
 # PSF model parameters : choose the PSF model desired
 PSF_MODEL           STR  PS_MODEL_PS1_V1
+PSPHOT.CR.NSIGMA.SOFTEN             F32   0.0025            # Softening parameter for weights
 
 PSF.TREND.MODE                      STR   MAP
-PSF.TREND.NX                        S32   1
-PSF.TREND.NY                        S32   1
+# PSF.TREND.MODE                      STR   POLY_CHEB
+# PSF.TREND.MODE                      STR   POLY_ORD
+PSF.TREND.NX                        S32   3
+PSF.TREND.NY                        S32   3
 
 MOMENTS_SN_MIN      F32   30.0
@@ -46,8 +51,9 @@
 PSF_SHAPE_NSIGMA     F32  3.0		 # max significance for shape variation
 PSF_MAX_CHI          F32  50.0		 # reject objects worse that this
-APTREND.ORDER.MAX    S32  1
+
+APTREND.ORDER.MAX    S32  3
 
 MEASURE.APTREND      BOOL TRUE           ### XXX for now, skip this (too many errors)
-# BREAK_POINT STR PASS1
+# BREAK_POINT STR BACKMDL
 
 DIAGNOSTIC.PLOTS                    METADATA
@@ -58,2 +64,8 @@
 
 USE_FOOTPRINTS                      BOOL  TRUE       	  # use new pmFootprint peak packaging
+
+
+DIFF	METADATA
+	BACKGROUND.XBIN	    S32  32            # size of background superpixels
+	BACKGROUND.YBIN	    S32  32            # size of background superpixels
+END
Index: branches/bills_081204/ippconfig/megacam/dvo.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/megacam/dvo.config	(revision 20530)
+++ branches/bills_081204/ippconfig/megacam/dvo.config	(revision 20890)
@@ -2,4 +2,7 @@
 # location of DVO database tables
 CATDIR			/data/alala.0/eugene/isp/catdir
+
+# word used to identify photcodes belonging to this camera
+MOSAICNAME		MEGACAM
 
 # keywords used by DVO to interpret the headers
Index: branches/bills_081204/ippconfig/recipes/filerules-mef.mdc
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/filerules-mef.mdc	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/filerules-mef.mdc	(revision 20890)
@@ -193,4 +193,5 @@
 PSASTRO.STATS           OUTPUT {OUTPUT}.stats                    STATS     NONE       FPA        TRUE      NONE
 PSASTRO.CONFIG          OUTPUT {OUTPUT}.psastro.mdc              TEXT      NONE       FPA        TRUE      NONE
+PSASTRO.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK      COMP_MASK  CHIP       TRUE      NONE
 	      									     
 PSWARP.OUTPUT         	OUTPUT {OUTPUT}.fits                     IMAGE     COMP_IMG   FPA        TRUE      NONE
Index: branches/bills_081204/ippconfig/recipes/filerules-simple.mdc
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/filerules-simple.mdc	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/filerules-simple.mdc	(revision 20890)
@@ -148,4 +148,5 @@
 PSASTRO.STATS       	OUTPUT {OUTPUT}.stats        STATS     NONE       FPA        TRUE      NONE
 PSASTRO.CONFIG          OUTPUT {OUTPUT}.psastro.mdc  TEXT      NONE       FPA        TRUE      NONE
+PSASTRO.OUTPUT.MASK     OUTPUT {OUTPUT}.mk.fits      MASK      COMP_MASK  CHIP       TRUE      NONE
 						     
 PSWARP.OUTPUT       	OUTPUT {OUTPUT}.fits         IMAGE     NONE       FPA        TRUE      NONE
Index: branches/bills_081204/ippconfig/recipes/filerules-split.mdc
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/filerules-split.mdc	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/filerules-split.mdc	(revision 20890)
@@ -53,4 +53,6 @@
 PSASTRO.INPUT.CMP       INPUT    @FILES        CHIP       CMP
 PSASTRO.INPUT.CMF       INPUT    @FILES        CHIP       CMF
+PSASTRO.INPUT.MASK      INPUT    @FILES        CHIP       MASK
+PSASTRO.REFMASK         INPUT    @DETDB        CHIP       MASK
 PSASTRO.WCS             INPUT    @FILES        CHIP       WCS
 PSASTRO.MODEL           INPUT    @DETDB        FPA        ASTROM
@@ -159,4 +161,5 @@
 PSASTRO.STATS           OUTPUT {OUTPUT}.stats                    STATS     NONE       FPA        TRUE      NONE
 PSASTRO.CONFIG          OUTPUT {OUTPUT}.psastro.mdc              TEXT      NONE       FPA        TRUE      NONE
+PSASTRO.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK      COMP_MASK  CHIP       TRUE      NONE
 		        									        
 PSWARP.OUTPUT           OUTPUT {OUTPUT}.fits                     IMAGE     COMP_IMG   FPA        TRUE      NONE
@@ -192,8 +195,8 @@
 PPSTAMP.CHIP            OUTPUT {OUTPUT}.ch.fits                  IMAGE     NONE       CHIP       FALSE     MEF
 		        									        
-PPSIM.OUTPUT            OUTPUT {OUTPUT}.fits                     IMAGE     NONE       FPA        TRUE      MEF
+PPSIM.OUTPUT            OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      SPLIT
 PPSIM.FAKE.CHIP        	OUTPUT {OUTPUT}.fake.fits     		 IMAGE     NONE       FPA        TRUE      SIMPLE
 PPSIM.FORCE.CHIP       	OUTPUT {OUTPUT}.force.fits     		 IMAGE     NONE       FPA        TRUE      SIMPLE
-PPSIM.SOURCES           OUTPUT {OUTPUT}.cmf                      CMF       NONE       FPA        TRUE      MEF
+PPSIM.SOURCES           OUTPUT {OUTPUT}.{CHIP.NAME}.cmf          CMF       NONE       CHIP       TRUE      SPLIT
 PPSIM.FAKE.SOURCES     	OUTPUT {OUTPUT}.fake.cmf                 CMF       NONE       FPA        TRUE      NONE
 PPSIM.FORCE.SOURCES    	OUTPUT {OUTPUT}.force.cmf                CMF       NONE       FPA        TRUE      NONE
Index: branches/bills_081204/ippconfig/recipes/masks.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/masks.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/masks.config	(revision 20890)
@@ -23,5 +23,5 @@
 CR		U8	0x20		# Pixel contains a cosmic ray
 GHOST		U8	0x20		# Pixel contains an optical ghost
-
+STREAK		U8	0x20		# Pixel contains a streak
 
 ###### The following values are what I'm aiming to have in the long term (PAP, 2008-09-09)
@@ -29,20 +29,21 @@
 #### Detector-intrinsic; these will default to DETECTOR
 ###DETECTOR	U8	0x01		# Something is wrong with the detector
-###DARK		U8	0x02		# Pixel doesn't dark-subtract properly
+###DARK		U8	0x01		# Pixel doesn't dark-subtract properly
 ###FLAT		U8	0x01		# Pixel doesn't flat-field properly
 ###BLANK	U8	0x01		# Pixel doesn't contain valid data
 ###
 #### Detector-extrinsic; these will default to RANGE
-###RANGE	U8	0x04		# Pixel is out-of-range of linearity
-###SAT		U8	0x04		# Pixel is saturated
-###LOW		U8	0x04		# Pixel is low
+###RANGE	U8	0x02		# Pixel is out-of-range of linearity
+###SAT		U8	0x02		# Pixel is saturated
+###LOW		U8	0x02		# Pixel is low
 ###
 #### Convolution: pixels that touched a bad pixel
-###POOR		U8	0x08		# Pixel is poor after convolution with a bad pixel
-###CONV		U8	0x10		# Pixel is bad after convolution with a bad pixel
+###POOR		U8	0x04		# Pixel is poor after convolution with a bad pixel
+###CONV		U8	0x08		# Pixel is bad after convolution with a bad pixel
 ###
 #### Objects: real signal that should be ignored
-###CR		U8	0x20		# Pixel contains a cosmic ray
-###GHOST	U8	0x40		# Pixel contains an optical ghost
+###CR		U8	0x10		# Pixel contains a cosmic ray
+###GHOST	U8	0x20		# Pixel contains an optical ghost
+###STREAK	U8	0x40		# Pixel contains a streak
 ###
-####MARK		U8	0x80		# RESERVED for MARK
+####MARK	U8	0x80		# RESERVED for MARK
Index: branches/bills_081204/ippconfig/recipes/ppImage.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/ppImage.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/ppImage.config	(revision 20890)
@@ -8,4 +8,5 @@
 BIAS             BOOL    TRUE            # Bias subtraction
 DARK             BOOL    TRUE            # Dark subtraction
+REMNANCE         BOOL    FALSE           # Remnance masking
 SHUTTER          BOOL    FALSE           # Shutter correction
 FLAT             BOOL    TRUE            # Flat-field normalisation
@@ -64,4 +65,8 @@
 FRINGE.KEEP     F32     0.5             # Minimum fraction to keep in fringe solution
 
+# Remnance detection options
+REMNANCE.SIZE		S32	30		# Size for remnance detection
+REMNANCE.THRESH		F32	2.0		# Threshold for remnance detection
+
 # binned output image options
 BIN1.XBIN               S32      4
@@ -117,4 +122,9 @@
 END
 
+# Standard chip processing for areas with high stellar density
+CHIP_DENSE_STARS	METADATA
+END
+
+
 # No operation except rebinning
 PPIMAGE_BIN        METADATA
@@ -537,4 +547,32 @@
     FLAT METADATA
       DETTYPE STR FLAT_RAW
+    END
+  END   
+END
+
+# Overscan, bias, dark, shutter, flat-field, fringe, photom
+PPIMAGE_FLATTEST   METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.WEIGHT.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    TRUE            # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    TRUE            # Mask bad pixels
+  FRINGE           BOOL    TRUE            # Fringe subtraction
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  PHOTOM           BOOL    TRUE            # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+
+  DETREND.CONSTRAINTS  METADATA
+    FLAT METADATA
+      DETTYPE STR FLATTEST
     END
   END   
Index: branches/bills_081204/ippconfig/recipes/ppStack.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/ppStack.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/ppStack.config	(revision 20890)
@@ -7,7 +7,8 @@
 MASK.BAD	STR	BLANK		# Mask value to give bad pixels
 MASK.POOR	STR	POOR.WARP	# Mask value to give poor pixels
-POOR.FRACTION	F32	1.0		# Maximum fraction of bad weight for poor pixels
+POOR.FRACTION	F32	0.01		# Maximum fraction of bad weight for poor pixels
 THRESHOLD.MASK	F32	0.5		# Threshold for mask deconvolution (0..1)
-IMAGE.REJ	F32	0.2		# Rejected pixel fraction threshold for rejecting entire image
+IMAGE.REJ	F32	0.1		# Rejected pixel fraction threshold for rejecting entire image
+MATCH.REJ	F32	2.5		# Rejection threshold for chi^2 values from matching
 ROWS		S32	64		# Number of rows to read at once
 VARIANCE	BOOL	TRUE		# Use variance in rejection?
@@ -46,2 +47,9 @@
 STACK	METADATA
 END
+
+
+# Debugging
+DEBUG	METADATA
+	TEMP.DIR	STR	./		# Directory for temporary images
+	TEMP.DELETE	BOOL	FALSE		# Delete temporary files on completion?
+END
Index: branches/bills_081204/ippconfig/recipes/ppSub.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/ppSub.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/ppSub.config	(revision 20890)
@@ -2,10 +2,11 @@
 
 KERNEL.TYPE	STR	ISIS		# Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS)
-KERNEL.SIZE     S32	35		# Kernel half-size (pixels)
-SPATIAL.ORDER   S32	1		# Spatial polynomial order
+KERNEL.SIZE     S32	15		# Kernel half-size (pixels)
+SPATIAL.ORDER   S32	0		# Spatial polynomial order
 REGION.SIZE	F32	0		# Iso-kernel region size (pixels)
-STAMP.SPACING   F32	400		# Typical spacing between stamps (pixels)
-STAMP.FOOTPRINT S32	35		# Size of stamps (pixels)
-STAMP.THRESHOLD F32	0		# Flux threshold for stamps (ADU)
+STAMP.SPACING   F32	200		# Typical spacing between stamps (pixels)
+STAMP.FOOTPRINT S32	20		# Size of stamps (pixels)
+STAMP.THRESHOLD F32	5		# Flux threshold for stamps (stdev above background)
+STRIDE		S32	100		# Size of convolution patches (pixels)
 ITER            S32	1		# Number of rejection iterations
 REJ             F32	2.5		# Rejection level (std dev)
@@ -14,5 +15,5 @@
 MASK.BAD        STR	BLANK		# Mask value to give bad pixels
 MASK.POOR       STR	POOR.WARP	# Mask value to give poor pixels
-POOR.FRACTION	F32	0.25		# Maximum fraction of bad weight for poor pixels
+POOR.FRACTION	F32	0.10		# Maximum fraction of bad weight for poor pixels
 BADFRAC		F32	0.95		# Maximum fraction of bad pixels
 @ISIS.WIDTHS	F32	1 3 5 7		# Gaussian FWHMs for ISIS kernels
@@ -45,8 +46,9 @@
 # Recipe overrides for STACK
 STACK	METADATA
-	KERNEL.SIZE     S32	20	# Kernel half-size (pixels)
-	STAMP.SPACING   F32	400	# Typical spacing between stamps (pixels)	
-	STAMP.FOOTPRINT S32	20	# Size of stamps (pixels)
-	POOR.FRACTION	F32	0.9	# Maximum fraction of bad weight for poor pixels
+	KERNEL.SIZE     S32	15	# Kernel half-size (pixels)
+	STAMP.SPACING   F32	300	# Typical spacing between stamps (pixels)	
+	STAMP.FOOTPRINT S32	15	# Size of stamps (pixels)
+	POOR.FRACTION	F32	0.1	# Maximum fraction of bad weight for poor pixels
+	SPATIAL.ORDER   S32	0	# Spatial polynomial order
 END
 
Index: branches/bills_081204/ippconfig/recipes/psastro.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/psastro.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/psastro.config	(revision 20890)
@@ -112,4 +112,16 @@
 PSASTRO.MOSAIC.GRADIENT.NY    S32      2   # number of y-dir cells per chip
 
+REFSTAR_MASK                    BOOL FALSE
+REFSTAR_MASK_REGIONS            BOOL FALSE
+REFSTAR_MASK_MAX_MAG            F32 -15.0
+REFSTAR_MASK_SATSTAR_MAG_MAX    F32 -17.0
+REFSTAR_MASK_SATSTAR_MAG_SLOPE  F32  10.0
+REFSTAR_MASK_SATSTAR_POS_ZERO   F32   0.0
+REFSTAR_MASK_SATSPIKE_MAG_SLOPE F32  80.0
+REFSTAR_MASK_SATSPIKE_MAG_MAX   F32 -17.0
+REFSTAR_MASK_SATSPIKE_WIDTH     F32  10.0
+REFSTAR_MASK_BLEED_MAG_MAX      F32 -15.0
+REFSTAR_MASK_BLEED_MAG_SLOPE    F32   5.0
+
 # 2MASS default configuration (use 2MASS_J as ref magnitude)
 PSASTRO.CATDIR                STR      2MASS
@@ -144,6 +156,6 @@
 PSASTRO.USE.MODEL             BOOL     FALSE
 
-PSASTRO.PIXEL.TOLERANCE       F32      0.1
-PSASTRO.ANGLE.TOLERANCE       F32      0.1
+PSASTRO.PIXEL.TOLERANCE       F32      3.0
+PSASTRO.ANGLE.TOLERANCE       F32      1.0
 
 PSASTRO.FINE METADATA
Index: branches/bills_081204/ippconfig/recipes/psphot.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/psphot.config	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/psphot.config	(revision 20890)
@@ -259,5 +259,5 @@
 PSPHOT.EXT.NSIGMA.LIMIT             F32   4.0  # sources with extNsigma greater that this get tagged as likely extended sources
 PSPHOT.CR.GROW                      S32   1               # Number of pixels to grow CR mask
-
+PSPHOT.CR.NSIGMA.SOFTEN             F32   0.00            # Softening parameter for weights
 
 
@@ -266,4 +266,14 @@
 END
 
+# Recipe overrides for PSPHOT in sdssmosaic
+PSPHOT_NOAP_PSF METADATA
+END
+
+PSPHOT_AP_PSF METADATA
+END
+
+PSPHOT_AP_NOPSF METADATA
+END
+
 # Recipe overrides for STACK
 STACK	METADATA
Index: branches/bills_081204/ippconfig/recipes/reductionClasses.mdc
===================================================================
--- branches/cnb_branch_20081104/ippconfig/recipes/reductionClasses.mdc	(revision 20530)
+++ branches/bills_081204/ippconfig/recipes/reductionClasses.mdc	(revision 20890)
@@ -166,5 +166,5 @@
 # photometry to make flat-field correction (apply flat_raw)
 FLATTEST	METADATA
-	CHIP_PPIMAGE	STR	CHIP
+	CHIP_PPIMAGE	STR	PPIMAGE_FLATTEST
 	CHIP_PSPHOT	STR	CHIP
 	STACK_PPSTACK	STR	STACK
@@ -219,2 +219,18 @@
 	FAKEPHOT	STR	FAKEPHOT
 END
+
+
+# Intended for areas with high stellar density
+DENSE_STARS	METADATA
+	CHIP_PPIMAGE	STR	CHIP_DENSE_STARS
+	CHIP_PSPHOT	STR	CHIP
+	STACK_PPSTACK	STR	STACK
+	STACK_PPSUB	STR	STACK
+	STACK_PSPHOT	STR	STACK
+	DIFF_PPSUB	STR	DIFF
+	DIFF_PSPHOT	STR	DIFF
+	JPEG_BIN1	STR	PPIMAGE_J1
+	JPEG_BIN2	STR	PPIMAGE_J2
+	FAKEPHOT	STR	FAKEPHOT
+	ADDSTAR		STR	ADDSTAR
+END
Index: branches/bills_081204/ippconfig/sdssmosaic/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/Makefile.am	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/Makefile.am	(revision 20890)
@@ -6,5 +6,5 @@
 	dvo.layout \
 	camera.config \
-	format.config \
+	format_split.config \
 	format_mef.config \
 	cmp.config \
Index: branches/bills_081204/ippconfig/sdssmosaic/camera.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/camera.config	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/camera.config	(revision 20890)
@@ -8,15 +8,40 @@
 FORMATS         METADATA
 	MEF		STR	sdssmosaic/format_mef.config	# MEF version, required for psastro
-        SDSSmosaic	STR     sdssmosaic/format.config
+	SDSSmosaic	STR     sdssmosaic/format_split.config  # New (possibly) split format with 30 different chips
+        # SDSSmosaic	STR     sdssmosaic/format.config  	# Old split format with per-chip gains rolled up in DEPEND
 END
  
 # Description of camera --- all the chips and the cells that comprise them
 FPA     METADATA
-	CHIP1	STR	Cell
-	CHIP2	STR	Cell
-	CHIP3	STR	Cell
-	CHIP4	STR	Cell
-	CHIP5	STR	Cell
-	CHIP6	STR	Cell
+	CHIP11	STR	CELL11
+	CHIP12	STR	CELL12
+	CHIP13	STR	CELL13
+	CHIP14	STR	CELL14
+	CHIP15	STR	CELL15
+	CHIP16	STR	CELL16
+	CHIP21	STR	CELL21
+	CHIP22	STR	CELL22
+	CHIP23	STR	CELL23
+	CHIP24	STR	CELL24
+	CHIP25	STR	CELL25
+	CHIP26	STR	CELL26
+	CHIP31	STR	CELL31
+	CHIP32	STR	CELL32
+	CHIP33	STR	CELL33
+	CHIP34	STR	CELL34
+	CHIP35	STR	CELL35
+	CHIP36	STR	CELL36
+	CHIP41	STR	CELL41
+	CHIP42	STR	CELL42
+	CHIP43	STR	CELL43
+	CHIP44	STR	CELL44
+	CHIP45	STR	CELL45
+	CHIP46	STR	CELL46
+	CHIP51	STR	CELL51
+	CHIP52	STR	CELL52
+	CHIP53	STR	CELL53
+	CHIP54	STR	CELL54
+	CHIP55	STR	CELL55
+	CHIP56	STR	CELL56
 END
 
@@ -35,5 +60,5 @@
 END
 
-DVO.CAMERADIR	STR	sdss		# Camera directory for DVO
+DVO.CAMERADIR	STR	sdssmosaic	# Camera directory for DVO
 
 # convert supplied FPA.OBSTYPE values to abstract exptype names
Index: branches/bills_081204/ippconfig/sdssmosaic/dvo.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/dvo.config	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/dvo.config	(revision 20890)
@@ -1,6 +1,8 @@
-DVO.CAMERADIR		/home/jester/IPP/ippconfig/sdssmosaic
+# DVO.CAMERADIR		/home/jester/IPP/ippconfig/sdssmosaic
+# Previous one not needed any more?
 
 # location of DVO database tables
-CATDIR			/disk1/jester/data/SDSS/stripe82/coadd/catdir
+CATDIR			/IPP/data/SDSS/stripe82/coadd/prod/catdir_default
+MOSAICNAME		SDSSmosaic
 
 # keywords used by DVO to interpret the headers
@@ -43,5 +45,5 @@
 
 # correlation radius (arcseconds)
-ADDSTAR_RADIUS		2.0
+ADDSTAR_RADIUS		1.0
 
 # scaled correlation radius 
@@ -53,4 +55,5 @@
 STAR_CHISQ              10.0   # mark stars POOR if Xm > STAR_CHISQ
 STAR_TOOFEW              3     # mark star FEW if N(good) < STAR_TOOFEW
+GRID_TOOFEW             10     # keep grid FROZEN if N(good) < GRID_TOOFEW
 IMAGE_TOOFEW            10     # mark image FEW if N(good) < IMAGE_TOOFEW
 IMAGE_GOOD_FRACTION     0.05   # mark image FEW if N(good) < IMAGE_GOOD_FRACTION * Nstars
Index: branches/bills_081204/ippconfig/sdssmosaic/format.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/format.config	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/format.config	(revision 20890)
@@ -105,6 +105,41 @@
 	INSTRUME	STR	SDSSmosaic
 	FPA.INSTRUMENT	STR	INSTRUME
-# The gain is only roughly correct...
-	CELL.GAIN	F32	5.0
+	CELL.GAIN.ORIGIN STR	VALUE
+	CELL.GAIN	 F32	5.0
+# The following doesn't work - don't know why. Use format_split.config
+# instead to get correct per-chip gains.
+# 	CELL.GAIN.DEPEND STR	FPA.DETECTOR
+# 	CELL.GAIN	METADATA
+#                 11      F32     4.71
+#                 12      F32     4.6
+#                 13      F32     4.72
+#                 14      F32     4.76
+#                 15      F32     4.725
+#                 16      F32     4.895
+#                 21      F32     5.165
+#                 22      F32     6.565
+#                 23      F32     4.86
+#                 24      F32     4.885
+#                 25      F32     4.64
+#                 26      F32     4.76
+#                 31      F32     1.62
+#                 32      F32     1.595
+#                 33      F32     1.58
+#                 34      F32     1.6
+#                 35      F32     1.47
+#                 36      F32     2.17
+#                 41      F32     4.745
+#                 42      F32     5.155
+#                 43      F32     4.885
+#                 44      F32     4.775
+#                 45      F32     3.48
+#                 46      F32     4.69
+#                 51      F32     4.745
+#                 52      F32     5.155
+#                 53      F32     4.885
+#                 54      F32     4.775
+#                 55      F32     3.48
+#                 56      F32     4.69		
+# END
 # readnoise is from http://www.sdss.org/dr6/instruments/imager/index.html
 # Note that I have emails from Jim Gunn saying dark noise *is* important...
Index: branches/bills_081204/ippconfig/sdssmosaic/format_split.config
===================================================================
--- branches/bills_081204/ippconfig/sdssmosaic/format_split.config	(revision 20890)
+++ branches/bills_081204/ippconfig/sdssmosaic/format_split.config	(revision 20890)
@@ -0,0 +1,412 @@
+# SDSSmosaic camera, starting from SDSS from Pan-STARRS Imaging Sky Probe
+# Sebastian Jester jester at mpia.de August 7, 2007
+#
+# Renamed to format_split.config to specify different chips for the actual 30 different chips following megacam/format_split.config
+# Here we want 60 cells, corresponding to the 6 camcols x 5 chips (one per filter) x 2 amps (which can have different gains)
+# October 2008 
+
+# How to identify this type
+RULE	METADATA
+	SIMPLE		BOOL	TRUE
+#	NAXIS		S32	2
+	ORIGIN		STR	SDSS
+	TELESCOP	STR	2.5m
+END
+
+# How to read this data
+FILE	METADATA
+	PHU		STR	CHIP	# The FITS file represents one
+					# camcol = chip, 6 of which
+					# make the FPA in one filter,
+					# times 5 to make all filters.
+	EXTENSIONS	STR	NONE	# There are no extensions
+	FPA.OBS		STR	FRAME	# A PHU keyword for unique identifier within 
+					# the hierarchy level - just needs to be the same
+					# for all 6 files belonging to the same "exposure"
+# was	CHIP.NAME	STR	CAMCOL	# Distinguishes chip name
+	CHIP.NAME	STR	CCDLOC	# Distinguishes chip name
+	CONTENT		STR	CCDLOC
+ # Is the following line the origin of the weirdly named output files?
+	CONTENT.RULE	STR	{CHIP.NUM} # How to derive the CONTENT when writing
+END
+
+# What's in the FITS file?
+# CONTENTS	STR	Chip:Cell:amplifier
+CONTENTS	METADATA
+	# Content name, chipType
+	11		STR	CHIP11:Chip11
+	12		STR	CHIP12:Chip12
+	13		STR	CHIP13:Chip13
+	14		STR	CHIP14:Chip14
+	15		STR	CHIP15:Chip15
+	16		STR	CHIP16:Chip16
+	21		STR	CHIP21:Chip21
+	22		STR	CHIP22:Chip22
+	23		STR	CHIP23:Chip23
+	24		STR	CHIP24:Chip24
+	25		STR	CHIP25:Chip25
+	26		STR	CHIP26:Chip26
+	31		STR	CHIP31:Chip31
+	32		STR	CHIP32:Chip32
+	33		STR	CHIP33:Chip33
+	34		STR	CHIP34:Chip34
+	35		STR	CHIP35:Chip35
+	36		STR	CHIP36:Chip36
+	41		STR	CHIP41:Chip41
+	42		STR	CHIP42:Chip42
+	43		STR	CHIP43:Chip43
+	44		STR	CHIP44:Chip44
+	45		STR	CHIP45:Chip45
+	46		STR	CHIP46:Chip46
+	51		STR	CHIP51:Chip51
+	52		STR	CHIP52:Chip52
+	53		STR	CHIP53:Chip53
+	54		STR	CHIP54:Chip54
+	55		STR	CHIP55:Chip55
+	56		STR	CHIP56:Chip56
+END
+	
+# Specify the chips - only u3 has 2 amps, and I don't even know if the gains differ.
+CHIPS		METADATA
+	# Chip type, cellName:cellType
+	Chip11	STR	CELL11:Amp11
+	Chip12	STR	CELL12:Amp12
+	Chip13	STR	CELL13:Amp13
+	Chip14	STR	CELL14:Amp14
+	Chip15	STR	CELL15:Amp15
+	Chip16	STR	CELL16:Amp16
+	Chip21	STR	CELL21:Amp21
+	Chip22	STR	CELL22:Amp22
+	Chip23	STR	CELL23:Amp23
+	Chip24	STR	CELL24:Amp24
+	Chip25	STR	CELL25:Amp25
+	Chip26	STR	CELL26:Amp26
+	Chip31	STR	CELL31:Amp31
+	Chip32	STR	CELL32:Amp32
+	Chip33	STR	CELL33:Amp33
+	Chip34	STR	CELL34:Amp34
+	Chip35	STR	CELL35:Amp35
+	Chip36	STR	CELL36:Amp36
+	Chip41	STR	CELL41:Amp41
+	Chip42	STR	CELL42:Amp42
+	Chip43	STR	CELL43:Amp43
+	Chip44	STR	CELL44:Amp44
+	Chip45	STR	CELL45:Amp45
+	Chip46	STR	CELL46:Amp46
+	Chip51	STR	CELL51:Amp51
+	Chip52	STR	CELL52:Amp52
+	Chip53	STR	CELL53:Amp53
+	Chip54	STR	CELL54:Amp54
+	Chip55	STR	CELL55:Amp55
+	Chip56	STR	CELL56:Amp56
+END
+
+
+# Specify the cell data
+# Gain is from ? XXX
+
+# readnoise is from http://www.sdss.org/dr6/instruments/imager/index.html and just one number for all chips.
+# Note that I have emails from Jim Gunn saying dark noise *is* important...
+CELLS	METADATA
+	Amp11	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.71
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp12	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.6
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp13	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.72
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp14	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.76
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp15	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.725
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp16	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.895
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp21	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	5.165
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp22	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	6.565
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp23	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.86
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp24	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.885
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp25	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.64
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp26	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.76
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp31	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	1.62
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp32	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	1.595
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp33	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	1.58
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp34	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	1.6
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp35	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	1.47
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp36	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	2.17
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp41	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.745
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp42	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	5.155
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp43	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.885
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp44	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.775
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp45	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	3.48
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp46	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.69
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp51	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.745
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp52	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	5.155
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp53	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.885
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp54	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.775
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp55	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	3.48
+		CELL.X0			S32	0 # or 1?
+	END
+	Amp56	METADATA
+		CELL.TRIMSEC.SOURCE	STR	VALUE
+		CELL.TRIMSEC		STR	[1:2048,1:1361]
+		CELL.GAIN.SOURCE	STR	VALUE
+		CELL.GAIN		F32	4.69
+		CELL.X0			S32	0 # or 1?
+	END
+END
+
+# How to translate PS concepts into FITS headers
+TRANSLATION	METADATA
+	FPA.TELESCOPE	STR	TELESCOP
+# FPA.INSTRUMENT actually needs to be identical to the CAMERA, 
+# so I'm moving it from here down to the defaults to make sure that
+# this is ture
+#	FPA.INSTRUMENT	STR	ORIGIN
+	FPA.DETECTOR	STR	CCDLOC
+#	FPA.AIRMASS	STR	AIRMASS
+	FPA.OBSTYPE	STR	FLAVOR
+	FPA.OBJECT	STR	OBJECT
+	FPA.FILTERID	STR	FILTER
+	FPA.FILTER	STR	FILTER
+	FPA.POSANGLE	STR	IPA
+	FPA.RA		STR	RA
+	FPA.DEC		STR	DEC
+	FPA.RADECSYS	STR	RADECSYS
+#	FPA.FOCUS	STR	TELFOCUS
+	FPA.TIME    	STR     DATE-OBS TAIHMS # Two values are interpreted as date and time
+	FPA.TIMESYS	STR	TIMESYS
+	FPA.ALT		STR	ALT
+	FPA.AZ		STR	AZ
+#	FPA.TEMP	STR	CCDTEMP
+	FPA.EXPOSURE	STR	EXPTIME
+	CHIP.TEMP	STR	CCDTEMP
+	CELL.EXPOSURE	STR	EXPTIME
+	CELL.DARKTIME	STR	EXPTIME
+	CELL.TIME	STR	DATE-OBS TAIHMS
+#	CELL.GAIN	STR	GAIN
+#	CELL.READNOISE	STR	RDNOISE
+#	CELL.SATURATION	STR	SATURATE	### Currently set to 0 ???
+#	CELL.TIMESYS	STR	TIMESYS
+	CELL.XBIN	STR	COLBIN
+	CELL.YBIN	STR	ROWBIN
+# these were used for some early data
+#	CELL.XBIN	STR	XBIN
+#	CELL.YBIN	STR	YBIN
+#	CELL.BAD	STR	BADLEVEL
+END
+
+# Default PS concepts that may be specified by value
+DEFAULTS	METADATA
+# FPA.CAMERA contains the name of the camera according to the config system (i.e. SDSSmosaic)
+# FPA.INSTRUMENT contains the name of the camera according to the fits headers (i.e. itself)
+# Normally, this will be in the header keyword INSTRUME but for this one, we override it to 
+# SDSSmosaic to avoid confusion between this and the non-mosaic SDSS configuration.
+	INSTRUME	STR	SDSSmosaic
+	FPA.INSTRUMENT	STR	INSTRUME
+# readnoise is from http://www.sdss.org/dr6/instruments/imager/index.html
+# Note that I have emails from Jim Gunn saying dark noise *is* important...
+       	CELL.READNOISE.SOURCE	STR	VALUE
+	CELL.READNOISE	F32	5.0
+# What are the units of readnoise? Electrons?
+	FPA.AIRMASS	F32	1.3
+	CHIP.XSIZE	S32	2048
+	CHIP.YSIZE	S32	1361
+	CELL.XSIZE	S32	2048
+	CELL.YSIZE	S32	1361
+#	FPA.TIMESYS	STR	UTC
+	CELL.SATURATION	F32	60000
+	CELL.READDIR	S32	1
+	CELL.TIMESYS	STR	UTC
+	CHIP.XPARITY	S32	1
+	CHIP.YPARITY	S32	1
+# Do we now need a CHIP.X0.DEPEND STR DETECTOR and a table of
+# locations? If so, again dr6/imager/index.html gives the necessary
+# parameters
+	CHIP.X0		S32	0
+	CHIP.Y0		S32	0
+	CELL.XPARITY	S32	1
+	CELL.YPARITY	S32	1
+	CELL.Y0		S32	0
+#	CELL.XBIN	S32	1
+#	CELL.YBIN	S32	1
+END
+
+FORMATS		METADATA
+	FPA.RA		STR	DEGREES
+	FPA.DEC		STR	DEGREES
+	FPA.TIME    	STR     SEPARATE YEAR.FIRST
+#	FPA.TIME	STR	MJD
+	CELL.TIME    	STR     SEPARATE YEAR.FIRST
+#	CELL.TIME	STR	MJD
+	CHIP.NUM	STR	FORTRAN
+END
+
+# PS Concepts to get from the database
+DATABASE	METADATA
+# None.
+END
Index: branches/bills_081204/ippconfig/sdssmosaic/psastro.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/psastro.config	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/psastro.config	(revision 20890)
@@ -72,5 +72,5 @@
 # in the PSASTRO.CATDIRS METADATA section. SYNTH.GRIZY is currently
 # the other option.
-PSASTRO.CATDIR STR 2MASS
+PSASTRO.CATDIR STR SYNTH.GRIZY
 
 # Mosaic Astrometry options
@@ -97,12 +97,36 @@
 PSASTRO.MOSAIC.CHIP.ORDER.N3  S32      1 # fit order (-1 means use default)
 
-# DVO.GETSTAR STR /home/kiawe/eugene/psconfig/dev.lin64/bin/getstar
-
 # we need to allow a loose fit if we are fitting to 2mass (50mas internal error -> 100mas limit?)
 # if we are fitting against quality digital data, we can require tighter constraints
 
-PSASTRO.MODEL.REF.CHIP        STR      XY33
+PSASTRO.MODEL.REF.CHIP        STR      CHIP31
 
 # Do I need to change this photcode now that I'm using 30 SDSS photcodes?
 DVO.GETSTAR.PHOTCODE        STR      g
-DVO.GETSTAR.MAG.MAX         F32      17.0
+
+PHOTCODE.DATA MULTI
+PHOTCODE.DATA METADATA
+  FILTER   STR r
+  ZEROPT   F32 24.0
+  PHOTCODE STR r
+END
+PHOTCODE.DATA METADATA
+  FILTER   STR i
+  ZEROPT   F32 22.0
+  PHOTCODE STR i
+END
+PHOTCODE.DATA METADATA
+  FILTER   STR u
+  ZEROPT   F32 23.8
+  PHOTCODE STR u
+END
+PHOTCODE.DATA METADATA
+  FILTER   STR z
+  ZEROPT   F32 21.9
+  PHOTCODE STR z
+END
+PHOTCODE.DATA METADATA
+  FILTER   STR g
+  ZEROPT   F32 24.4
+  PHOTCODE STR g
+END
Index: branches/bills_081204/ippconfig/sdssmosaic/psphot.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/psphot.config	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/psphot.config	(revision 20890)
@@ -1,47 +1,34 @@
 
 # turn these on to see specific outputs
+SAVE.OUTPUT	BOOL 	TRUE
 SAVE.BACKMDL	BOOL 	TRUE
+#SAVE.RESID	BOOL 	TRUE
 #SAVE.BACKGND	BOOL 	TRUE
-#SAVE.BACKSUB	BOOL 	TRUE
-#SAVE.RESID	BOOL 	TRUE
+SAVE.BACKSUB	BOOL 	TRUE
 SAVE.PSF	BOOL 	TRUE
-SAVE.PLOTS      BOOL    TRUE
+#LOAD.PSF	BOOL 	FALSE
+#SAVE.PLOTS     	BOOL    TRUE
 
-# image statistics parameters
-IMSTATS_NPIX        S32  3000    	 # number of pixels to use for sky estimate boxes:
+BACKGROUND.XBIN	    S32  256            # size of background superpixels
+BACKGROUND.YBIN	    S32  256            # size of background superpixels
 
-PSF_SN_LIM          F32  10              # minimum S/N for stars used for PSF model
+# image background parameters
+IMSTATS_NPIX        S32  10000    	 # number of pixels to use for sky estimate boxes:
+SKY_STAT            STR  FITTED_MEAN_V4  # statistic used to measure background
+SKY_CLIP_SIGMA      F32  2.0             # statistic used to measure background
+
+PEAKS_SMOOTH_SIGMA  F32   2.5            # smoothing kernel sigma in pixels
+PEAKS_NMAX          S32   5000           # on first pass, only keep NMAX peaks (0 == all)
+PEAKS_NSIGMA_LIMIT  F32   25.0            # peak significance threshold
+PEAKS_NSIGMA_LIMIT_2 F32  10.0             # peak significance threshold
+
+PSF_SN_LIM          F32  25              # minimum S/N for stars used for PSF model
 PSF_MAX_NSTARS      S32  300             # limit number of stars used for PSF model
+PSF_CLUMP_NX        S32   3               # subdivide image in to NX x NY regions for 
+PSF_CLUMP_NY        S32   3               # selecting PSF stars
+PSF_MOMENTS_RADIUS  F32  12               # calculate initial source moments with this radius
 
-PSF_MODEL         STR  PS_MODEL_QGAUSS
-
-MOMENTS_SN_MIN      F32   10.0
-#EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
-#FULL_FIT_SN_LIM      F32  50.0
-#AP_MIN_SN            F32  20.0
-# This is where aperture correction gets turned on and off - used to be false up to 2008-10-06
-MEASURE.APTREND	                    BOOL  FALSE
-PSF_CLUMP_NSIGMA   F32  2.5             # region of Sx,Sy plane to use for selecting PSF stars
-
-## # PSFTREND must be a 2D polynomial
-## # the specified values are ignored but define the active components of the polynomial
-## PSF.TREND.MASK  METADATA  
-##    NORDER_X         S32       3                # number of x orders
-##    NORDER_Y         S32       3                # number of y orders
-##    VAL_X00_Y00      F64       1                # polynomial coefficient
-## 
-##    VAL_X01_Y00      F64       1                # polynomial coefficient
-##    VAL_X00_Y01      F64       1                # polynomial coefficient
-## 
-##    VAL_X02_Y00      F64       1                # polynomial coefficient
-##    VAL_X01_Y01      F64       1                # polynomial coefficient
-##    VAL_X00_Y02      F64       1                # polynomial coefficient
-## 
-##    VAL_X03_Y00      F64       1                # polynomial coefficient
-##    VAL_X02_Y01      F64       1                # polynomial coefficient
-##    VAL_X01_Y02      F64       1                # polynomial coefficient
-##    VAL_X00_Y03      F64       1                # polynomial coefficient
-##    NELEMENTS        S32       10               # number of unmasked components
-## END  # folder for 4D polynomial
+# PSF model parameters : choose the PSF model desired
+PSF_MODEL           STR  PS_MODEL_PS1_V1
 
 # Use same default setting as in 2.6.1 (MAP is now global default)
@@ -50,36 +37,49 @@
 PSF.TREND.NY                        S32   0
 
-# Turn on variable PSF fitting - MAP tries to trade off between not having many stars and not having many bins.
-# Try nx, ny up to 3 or 5.
-# Note that MAP with NX, NY = 1 is the default in recipes/psphot.config now!
-# PSF.TREND.MODE	STR	MAP
-# PSF.TREND.NX		S32	5
-# PSF.TREND.NY		S32	5
+MOMENTS_SN_MIN      F32   30.0
+EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
+FULL_FIT_SN_LIM      F32  50.0
+AP_MIN_SN            F32  50.0
 
-XMIN F32 15
+OUTPUT.FORMAT       STR PS1_DEV_1
 
-PSF.RESIDUALS       BOOL true
-PSF.RESIDUALS.SPATIAL_ORDER S32 1
+PSF_SHAPE_NSIGMA     F32  3.0		 # max significance for shape variation
+PSF_MAX_CHI          F32  50.0		 # reject objects worse that this
+APTREND.ORDER.MAX    S32  1
 
-# The parameter BREAK_POINT stopped the analysis after first linear
-# fit to sources.  Should be NONE or commented out.
-# BREAK_POINT         STR  ENSEMBLE
-OUTPUT.FORMAT       STR  PS1_DEV_1
+MEASURE.APTREND      BOOL TRUE           ### XXX for now, skip this (too many errors)
+# BREAK_POINT STR PASS1
 
-PSPHOT.SUMMIT METADATA
- PEAKS_SMOOTH_SIGMA  F32  0.8 	   	 # peak significance threshold
- PEAKS_NSIGMA_LIMIT  F32  50.0 	   	 # peak significance threshold
- PEAKS_NMAX          S32  1000
- SAVE.RESID	BOOL 	FALSE
- PSF_SN_LIM          F32  25              # minimum S/N for stars used for PSF model
- MOMENTS_SN_MIN      F32   25.0
- PSF_MODEL         STR  PS_MODEL_PGAUSS
+PSF_REF_RADIUS      F32  18.7            # aperture magnitudes are scaled via 
+					 # curve-of-growth to this radius
+
+DIAGNOSTIC.PLOTS                    METADATA
+  IMAGE.BACKGROUND.CELL.HISTOGRAM   BOOL  FALSE
+  IMAGE.BACKGROUND.CELL.HISTOGRAM.X S32   32
+  IMAGE.BACKGROUND.CELL.HISTOGRAM.Y S32   15
 END
 
-# Turn on use of footprints to avoid objectcs being found in
-# diffraction spikes etc. of bright stars
-USE_FOOTPRINTS			BOOL	T	# use new pmFootprint peak packaging
+USE_FOOTPRINTS                      BOOL  TRUE       	  # use new pmFootprint peak packaging
 
-# Set aperture correction radius to same 7.4" as SDSS (see EDR paper)
-PSF_REF_RADIUS      F32  18.7            # aperture magnitudes are scaled via 
-					 # curve-of-growth to this radius
+# Default is noap, nopsf; following recipes turn on one or both of these
+
+PSPHOT_NOAP_PSF METADATA
+ PSF.TREND.MODE		STR	MAP
+ PSF.TREND.NX		S32	3
+ PSF.TREND.NY		S32	3
+ MEASURE.APTREND        BOOL  FALSE
+END
+
+PSPHOT_AP_NOPSF METADATA
+ PSF.TREND.MODE		STR	POLY_ORD
+ PSF.TREND.NX		S32	0
+ PSF.TREND.NY		S32	0
+ MEASURE.APTREND        BOOL  TRUE
+END
+
+PSPHOT_AP_PSF METADATA
+ PSF.TREND.MODE		STR	MAP
+ PSF.TREND.NX		S32	3
+ PSF.TREND.NY		S32	3
+ MEASURE.APTREND        BOOL  TRUE
+END
Index: branches/bills_081204/ippconfig/sdssmosaic/reductionClasses.mdc
===================================================================
--- branches/cnb_branch_20081104/ippconfig/sdssmosaic/reductionClasses.mdc	(revision 20530)
+++ branches/bills_081204/ippconfig/sdssmosaic/reductionClasses.mdc	(revision 20890)
@@ -5,5 +5,11 @@
 	# This one used to be called PROCESSED
 	SDSSphotonly	METADATA
-		CHIP		STR	PPIMAGE_OP_SOFT
+		CHIP_PPIMAGE	STR	PPIMAGE_OP_SOFT
+		CHIP_PSPHOT	STR	CHIP
+		STACK_PPSTACK	STR	STACK
+		STACK_PPSUB	STR	STACK
+		STACK_PSPHOT	STR	STACK
+		DIFF_PPSUB	STR	DIFF
+		DIFF_PSPHOT	STR	DIFF
  		JPEG_BIN1       STR     PPIMAGE_J1
  		JPEG_BIN2       STR     PPIMAGE_J2
@@ -13,5 +19,50 @@
         # photometry and astrometry for pre-detrended images
 	SDSSphotast	METADATA
-		CHIP		STR	PPIMAGE_OA_SOFT
+		CHIP_PPIMAGE	STR	PPIMAGE_OA_SOFT
+		CHIP_PSPHOT	STR	CHIP
+		STACK_PPSTACK	STR	STACK
+		STACK_PPSUB	STR	STACK
+		STACK_PSPHOT	STR	STACK
+		DIFF_PPSUB	STR	DIFF
+		DIFF_PSPHOT	STR	DIFF
+ 		JPEG_BIN1       STR     PPIMAGE_J1
+ 		JPEG_BIN2       STR     PPIMAGE_J2
+		FAKEPHOT	STR	FAKEPHOT
+		ADDSTAR		STR	ADDSTAR
+	END
+	SDSSphotast_ap	METADATA
+		CHIP_PPIMAGE	STR	PPIMAGE_OA_SOFT
+		CHIP_PSPHOT	STR	PSPHOT_AP_PSF
+		STACK_PPSTACK	STR	STACK
+		STACK_PPSUB	STR	STACK
+		STACK_PSPHOT	STR	STACK
+		DIFF_PPSUB	STR	DIFF
+		DIFF_PSPHOT	STR	DIFF
+ 		JPEG_BIN1       STR     PPIMAGE_J1
+ 		JPEG_BIN2       STR     PPIMAGE_J2
+		FAKEPHOT	STR	FAKEPHOT
+		ADDSTAR		STR	ADDSTAR
+	END
+	SDSSphotast_p	METADATA
+		CHIP_PPIMAGE	STR	PPIMAGE_OA_SOFT
+		CHIP_PSPHOT	STR	PSPHOT_NOAP_PSF
+		STACK_PPSTACK	STR	STACK
+		STACK_PPSUB	STR	STACK
+		STACK_PSPHOT	STR	STACK
+		DIFF_PPSUB	STR	DIFF
+		DIFF_PSPHOT	STR	DIFF
+ 		JPEG_BIN1       STR     PPIMAGE_J1
+ 		JPEG_BIN2       STR     PPIMAGE_J2
+		FAKEPHOT	STR	FAKEPHOT
+		ADDSTAR		STR	ADDSTAR
+	END
+	SDSSphotast_a	METADATA
+		CHIP_PPIMAGE	STR	PPIMAGE_OA_SOFT
+		CHIP_PSPHOT	STR	PSPHOT_AP_NOPSF
+		STACK_PPSTACK	STR	STACK
+		STACK_PPSUB	STR	STACK
+		STACK_PSPHOT	STR	STACK
+		DIFF_PPSUB	STR	DIFF
+		DIFF_PSPHOT	STR	DIFF
  		JPEG_BIN1       STR     PPIMAGE_J1
  		JPEG_BIN2       STR     PPIMAGE_J2
Index: branches/bills_081204/ippconfig/simmosaic/camera.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simmosaic/camera.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simmosaic/camera.config	(revision 20890)
@@ -3,5 +3,5 @@
 # File formats that we know about
 FORMATS         METADATA
-	TOGETHER STR	simmosaic/format_together.config
+	MEF     STR	simmosaic/format_together.config
 	SPLIT	STR	simmosaic/format_split.config
 END
@@ -65,5 +65,5 @@
 FITSTYPES       STR     recipes/fitstypes.mdc
 
-FILERULES       STR     recipes/filerules-mef.mdc
+FILERULES       STR     recipes/filerules-split.mdc
 
 EXTNAME.RULES	METADATA
Index: branches/bills_081204/ippconfig/simmosaic/psastro.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simmosaic/psastro.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simmosaic/psastro.config	(revision 20890)
@@ -1,4 +1,3 @@
-PSASTRO.CATDIR	STR	SYNTH.SIMTEST
-
-#PSASTRO.MOSAIC.GRADIENT.NX    S32     1   # number of x-dir cells per chip
-#PSASTRO.MOSAIC.GRADIENT.NY    S32     1   # number of y-dir cells per chip
+PSASTRO.CATDIR		STR	SYNTH.SIMTEST
+DVO.GETSTAR.PHOTCODE	STR	r
+PSASTRO.MAX.NSTAR	S32	50	# max stars accepted for fitting
Index: branches/bills_081204/ippconfig/simmosaic/psphot.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simmosaic/psphot.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simmosaic/psphot.config	(revision 20890)
@@ -1,5 +1,4 @@
 
 SAVE.BACKMDL	BOOL 	TRUE
-SAVE.BACKMDL.STDEV BOOL 	TRUE
 SAVE.BACKGND	BOOL 	TRUE
 SAVE.BACKSUB	BOOL 	TRUE
@@ -10,4 +9,9 @@
 
 IMSTATS_NPIX        S32  3000    	 # number of pixels to use for sky estimate boxes:
+SKY_INNER_RADIUS    F32    10            # square annulus for local sky measurement
+
+PSF_MOMENTS_RADIUS  F32    7               # calculate initial source moments with this radius
+
+
 
 #PSPHOT_TEST METADATA
@@ -20,9 +24,15 @@
 
 #PSF_SN_LIM          F32  20.0            # minimum S/N for stars used for PSF model
-MOMENTS_SN_MIN      F32  20.0            # min S/N to measure moments
+MOMENTS_SN_MIN      F32  10.0            # min S/N to measure moments
 FULL_FIT_SN_LIM     F32  100.0
 EXT_MIN_SN          F32  100.0           # fit galaxies above this S/N limit
 
-PEAKS_NSIGMA_LIMIT  F32  10.0 	   	 # peak significance threshold
+MOMENTS_SX_MAX                      F32   25.0
+MOMENTS_SY_MAX                      F32   25.0
+
+
+PEAKS_NSIGMA_LIMIT  F32  25.0 	   	 # peak significance threshold
+PEAKS_NSIGMA_LIMIT_2 F32  10.0 	   	 # peak significance threshold
+
 BREAK_POINT         STR  ENSEMBLE        # limit total processing for now (for speed)
 
Index: branches/bills_081204/ippconfig/simmosaic/rejections.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simmosaic/rejections.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simmosaic/rejections.config	(revision 20890)
@@ -1,2 +1,26 @@
+
+BIAS METADATA
+  FILTER      	     STR  *
+  EXPECTED    	     F32  0.0
+  IMFILE.MEAN 	     F32  2.0 # 2 sigma from expected mean
+  IMFILE.STDEV       F32 20.0 # per-image sigma of < 20.0 (supplied read-noise is 10.0)
+  IMFILE.MEANSTDEV   F32  0.0 
+  IMFILE.SKEWNESS    F32  0.0
+  IMFILE.KURTOSIS    F32  0.0
+  IMFILE.SNR         F32  0.0
+  IMFILE.BIN.STDEV   F32  0.5 # binned stdev < 0.5 (binning is 64x64 : 10.0 / 64.0 = 0.15)
+  IMFILE.BIN.SNR     F32  0.0
+  IMFILE.FLUX        F32  0.0
+  EXP.MEAN           F32  1.0 
+  EXP.STDEV          F32 20.0
+  EXP.MEANSTDEV      F32  0.0
+  EXP.SNR            F32  0.0
+  EXP.BIN.STDEV      F32  0.0
+  EXP.BIN.SNR        F32  0.0
+  EXP.FLUX           F32  0.0
+  ENSEMBLE.MEAN      F32  5.0
+  ENSEMBLE.STDEV     F32  5.0
+  ENSEMBLE.MEANSTDEV F32  0.0
+END
 
 DARK METADATA
@@ -19,6 +43,6 @@
   EXP.BIN.SNR        F32  0.0
   EXP.FLUX           F32  0.0
-  ENSEMBLE.MEAN      F32  3.0
-  ENSEMBLE.STDEV     F32  3.0
+  ENSEMBLE.MEAN      F32  5.0
+  ENSEMBLE.STDEV     F32  5.0
   ENSEMBLE.MEANSTDEV F32  0.0
 END
@@ -48,2 +72,7 @@
   ENSEMBLE.MEANSTDEV F32  0.0
 END
+
+
+ITERATION	BOOL	FALSE		# Are iterations permitted?
+MASTER		BOOL	TRUE		# Force stacks to be accepted as a master?
+
Index: branches/bills_081204/ippconfig/simtest/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simtest/Makefile.am	(revision 20530)
+++ branches/bills_081204/ippconfig/simtest/Makefile.am	(revision 20890)
@@ -13,5 +13,6 @@
 	dvo.layout \
 	ppSim.config \
-	ppSub.config
+	ppSub.config \
+	ppStack.config
 
 install_DATA = $(install_files)
Index: branches/bills_081204/ippconfig/simtest/camera.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simtest/camera.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simtest/camera.config	(revision 20890)
@@ -46,4 +46,5 @@
 	PPSIM		STR	simtest/ppSim.config	# ppSim details
 	PPSUB		STR	simtest/ppSub.config	# ppSub details
+	PPSTACK		STR	simtest/ppStack.config	# ppStack recipe overrides
 	PSASTRO		STR	simtest/psastro.config	# psastro details
 	REJECTIONS	STR	simtest/rejections.config # Rejection for detrend creation
@@ -74,3 +75,3 @@
 END
 
-PHOTCODE.RULE           STR     {DETECTOR}.{FILTER.ID}.{CHIP.N}		# Rule for generating photcode
+PHOTCODE.RULE           STR     {DETECTOR}.{FILTER.ID}.{CHIP.NAME}	# Rule for generating photcode
Index: branches/bills_081204/ippconfig/simtest/ppImage.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/simtest/ppImage.config	(revision 20530)
+++ branches/bills_081204/ippconfig/simtest/ppImage.config	(revision 20890)
@@ -274,2 +274,23 @@
 END
 
+
+# Standard chip processing
+CHIP   METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.WEIGHT.FITS BOOL    FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    TRUE            # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    TRUE           # Save chip-mosaic-ed image? 
+  CHIP.WEIGHT.FITS BOOL    TRUE           # Save chip-mosaic-ed image? 
+  OVERSCAN        BOOL    TRUE            # Overscan subtraction
+  BIAS            BOOL    TRUE            # Bias subtraction
+  DARK            BOOL    TRUE            # Dark subtraction
+  SHUTTER         BOOL    TRUE            # Shutter correction
+  FLAT            BOOL    TRUE            # Flat-field normalisation
+  MASK            BOOL    FALSE           # Mask bad pixels
+  FRINGE          BOOL    FALSE           # Fringe subtraction
+  PHOTOM          BOOL    TRUE		  # Source identification and photometry
+  ASTROM.CHIP     BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC   BOOL    FALSE		  # Astrometry for mosaic?
+END
+
Index: branches/bills_081204/ippconfig/simtest/ppStack.config
===================================================================
--- branches/bills_081204/ippconfig/simtest/ppStack.config	(revision 20890)
+++ branches/bills_081204/ippconfig/simtest/ppStack.config	(revision 20890)
@@ -0,0 +1,1 @@
+PSF.MODEL	STR	PS_MODEL_GAUSS	# Model for PSF generation
Index: branches/bills_081204/ippconfig/vysos5/psastro.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/vysos5/psastro.config	(revision 20530)
+++ branches/bills_081204/ippconfig/vysos5/psastro.config	(revision 20890)
@@ -70,5 +70,4 @@
 PSASTRO.CATDIR              STR      SYNTH.BRIGHT
 DVO.GETSTAR.PHOTCODE        STR      r
-DVO.GETSTAR.MAG.MAX         F32      12.0
 
 PSASTRO.PLOT.REF.MAG.MIN  F32 +5.0
Index: branches/bills_081204/ippconfig/vysos5/psphot.config
===================================================================
--- branches/cnb_branch_20081104/ippconfig/vysos5/psphot.config	(revision 20530)
+++ branches/bills_081204/ippconfig/vysos5/psphot.config	(revision 20890)
@@ -37,2 +37,3 @@
 PSF.TREND.NY                        S32   5
 
+PSF.RESIDUALS.SPATIAL_ORDER         S32   1               # fit spatial variations of the residuals at this order (0,1)
Index: branches/bills_081204/magic/README
===================================================================
--- branches/bills_081204/magic/README	(revision 20890)
+++ branches/bills_081204/magic/README	(revision 20890)
@@ -0,0 +1,15 @@
+To build The remove streaks portions of magic
+
+tar zxf ~sydney/magic.tgz
+tar xzf ~sydney/ssa-core-cpp.tgz
+
+cd ssa-core-cpp
+./configure
+make
+
+cd ../magic
+make install
+
+cd ../remove/src
+make install
+
Index: branches/bills_081204/magic/magic/makefile
===================================================================
--- branches/bills_081204/magic/magic/makefile	(revision 20890)
+++ branches/bills_081204/magic/magic/makefile	(revision 20890)
@@ -0,0 +1,32 @@
+#
+# This makefile is used in place of the one in Paul Sydneys' tarball
+# 
+# XXX integrate with the build system
+#
+TARGETS = RemoveStreaks
+SSA_SRC_DIR = ../ssa-core-cpp/src
+CFITSIO_DIR = ../cfitsio
+CFITSIO_DIR = $(PSCONFDIR)/$(PSCONFIG)
+INCLUDEDIRS = -I$(SSA_SRC_DIR) -I$(CFITSIO_DIR)/include
+# INCLUDEDIRS = -I$(SSA_SRC_DIR)
+
+CXX = c++
+CXXFLAGS = -Wall -O3 -DNDEBUG
+#CXXFLAGS = -Wall -g
+
+SRCS = $(TARGETS).cpp $(SSA_SRC_DIR)/math/Constants.cpp
+LIBS = -L$(CFITSIO_DIR) -lcfitsio
+LIBS = -L$(CFITSIO_DIR)/lib -lcfitsio
+# LIBS = -lcfitsio
+
+$(TARGETS) : $(SRCS)
+	   $(CXX) $(CXXFLAGS) $(INCLUDEDIRS) $(SRCS) $(LIBS) -o $@
+
+clean :
+	   - rm $(TARGETS)
+
+DESTDIR=$(PSCONFDIR)/$(PSCONFIG)
+
+install:	$(TARGETS)
+	cp RemoveStreaks $(DESTDIR)/bin
+	chmod 755  $(DESTDIR)/bin/RemoveStreaks
Index: branches/bills_081204/magic/remove/src/.cvsignore
===================================================================
--- branches/bills_081204/magic/remove/src/.cvsignore	(revision 20890)
+++ branches/bills_081204/magic/remove/src/.cvsignore	(revision 20890)
@@ -0,0 +1,3 @@
+streaksremove
+streaksreplace
+streakscompare
Index: branches/bills_081204/magic/remove/src/Line.c
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/Line.c	(revision 20530)
+++ branches/bills_081204/magic/remove/src/Line.c	(revision 20890)
@@ -39,4 +39,37 @@
            (tuple->y >= line->end.y &&
             tuple->y <= line->begin.y);
+}
+
+double DistanceSquared (Line* line, double x, double y)
+{
+    // Define U as the vector from the line segment start to end
+    // Define V as the vector from the line segment start to the tuple
+    // Define W as the vector from the line segment end to the tuple
+
+    double ux, uy, vx, vy, b, px, py;
+    ux = line->end.x - line->begin.x;
+    uy = line->end.y - line->begin.y;
+    
+    vx = x - line->begin.x;
+    vy = y - line->begin.y;
+    
+    double u_u = ux * ux + uy * uy;                         // (u . u)
+    double u_v = ux * vx + uy * vy;                         // (u . v)
+    
+    if (u_v <= 0) return vx * vx + vy * vy;                 // (v . v)
+    if (u_u <= u_v)
+    {
+        double wx =  x - line->end.x;
+        double wy =  y - line->end.y;
+        return wx * wx + wy * wy;                           // (w . w)
+    }
+    
+    // Compute P(b) is the base of the perpendicular dropped from tuple to
+    // the line
+    
+    b = u_v / u_u;
+    px = vx - b * ux;
+    py = vy - b * uy;
+    return px * px + py * py;                               // norm (p)
 }
 
@@ -280,11 +313,11 @@
         xEnd   = ceil  (x2 + xOffset) + 1.0;
 
-        for (x = xBegin; x != xEnd; ++x)
+        for (x = xBegin; x < xEnd; ++x)
         {
             yBegin = floor (yMid - yOffset);
             yEnd   = ceil  (yMid + yOffset) + 1.0;
-            for (y = yBegin; y != yEnd; ++y)
+            for (y = yBegin; y < yEnd; ++y)
             {
-                if ((x * x + y * y) <= halfWidth2)
+                if (DistanceSquared (line, x, y) <= halfWidth2)
                 {
                     pixel = psAlloc (sizeof(PixelPos));
@@ -327,11 +360,11 @@
         yEnd   = ceil  (y2 + yOffset) + 1.0;
         
-        for (y = yBegin; y != yEnd; ++y)
+        for (y = yBegin; y < yEnd; ++y)
         {
             xBegin = floor (xMid - xOffset);
             xEnd   = ceil  (xMid + xOffset) + 1.0;
-            for (x = xBegin; x != xEnd; ++x)
+            for (x = xBegin; x < xEnd; ++x)
             {
-                if ((x * x + y * y) <= halfWidth2)
+                if (DistanceSquared (line, x, y) <= halfWidth2)
                 {
                     pixel = psAlloc (sizeof(PixelPos));
Index: branches/bills_081204/magic/remove/src/Makefile.simple
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/Makefile.simple	(revision 20530)
+++ branches/bills_081204/magic/remove/src/Makefile.simple	(revision 20890)
@@ -1,5 +1,13 @@
 # skeleton Makefile for streaksremove
+#
+# The intent is that these programs may be built against an
+# IPP installation outside the IPP build enviornment.
 
-OBJECTS=    \
+COMMON_OBJECTS = \
+	streaksio.o \
+	streaksutil.o
+
+REMOVE_OBJECTS=    \
+    ${COMMON_OBJECTS} \
     streaksremove.o \
     streaksastrom.o \
@@ -8,15 +16,34 @@
     Line.o
 
-CFLAGS=`psmodules-config --cflags` -g -std=gnu99
+REPLACE_OBJECTS=    \
+    ${COMMON_OBJECTS} \
+    streaksreplace.o
+
+COMPARE_OBJECTS=    \
+    ${COMMON_OBJECTS} \
+    streakscompare.o
+
+STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1
+OPTFLAGS= -g
+CFLAGS=`psmodules-config --cflags` -std=gnu99 ${OPTFLAGS}
 LDFLAGS=`psmodules-config --libs`
 
-streaksremove:  ${OBJECTS}
+PROGRAMS= streaksremove streaksreplace streakscompare
 
-install:
-	cp streaksremove $(PSCONFDIR)/$(PSCONFIG)/bin
-	chmod 755  $(PSCONFDIR)/$(PSCONFIG)/bin/streaksremove
+all:	${PROGRAMS}
+
+streaksremove:  ${REMOVE_OBJECTS}
+
+streaksreplace:  ${REPLACE_OBJECTS}
+
+streakscompare:  ${COMPARE_OBJECTS}
+
+install:	${PROGRAMS}
+	install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streaksremove
+	install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streaksreplace
+	install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streakscompare
 
 clean:
-	rm -f *.o streaksremove    
+	rm -f *.o ${PROGRAMS}
     
 
Index: branches/bills_081204/magic/remove/src/streaks.h
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/streaks.h	(revision 20530)
+++ 	(revision )
@@ -1,86 +1,0 @@
-#ifndef STREAKS_H
-#define STREAKS_H
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include "string.h"
-#include "pslib.h"
-#include "psmodules.h"
-#include "nebclient.h"
-
-typedef struct {
-    int dummy;
-} Warps;
-
-#include "streaksastrom.h"
-
-typedef struct {
-    psString    resolved_name;
-    psString    name;
-    bool        inNebulous;
-    psFits      *fits;
-    int         nHDU;
-    psString    extname;
-    psMetadata  *header;
-    int         numZPlanes;
-    psImage     *image;
-    psArray     *imagecube;
-    pmFPAfile   *pmfile;
-    int         numCols;
-    int         numRows;
-} sFile;
-
-
-typedef enum {
-    IPP_STAGE_NONE = 0,
-    IPP_STAGE_RAW,
-    IPP_STAGE_CHIP,
-    IPP_STAGE_WARP,
-    IPP_STAGE_DIFF
-} ippStage;
-
-
-typedef struct {
-    pmConfig *config;
-    ippStage stage;
-    int     extnum;
-    int     nHDU;   // number of HDUs in the inputs (all must be equal)
-    sFile *inImage;
-    sFile *inMask;
-    sFile *inWeight;
-    sFile *outImage;
-    sFile *outMask;
-    sFile *outWeight;
-    sFile *recImage;
-    sFile *recMask;
-    sFile *recWeight;
-    psString class_id;
-    pmFPAfile  *inAstrom;
-    strkAstrom *astrom;
-    bool  bilevelAstrometry;
-    pmFPAview *view;
-    pmChip  *chip;  // current chip
-    pmCell  *cell;  // current cell
-    psImage *warpedPixels;
-    psVector    *tiles;
-    float   recoveryImageValue;
-    float   recoveryMaskValue;
-    float   recoveryWeightValue;
-} streakFiles;
-
-extern strkAstrom *streakSetAstrometry(strkAstrom *, pmFPA *, pmChip *, bool fromCell, psMetadata *md,
-    int numCols, int numRows);
-
-extern void computeWarpedPixels(streakFiles *sf);
-extern void streaksremoveExit(psString, int);
-
-#define CHIP_LEVEL_INPUT(_stage) ((_stage == IPP_STAGE_RAW) || (_stage == IPP_STAGE_CHIP))
-#define USE_SUPPLIED_ASTROM(_stage) CHIP_LEVEL_INPUT(_stage)
-
-#define SFILE_IS_IMAGE(_sfile) (_sfile->image || _sfile->imagecube)
-#define IN_NEBULOUS(_filename) (!strncasecmp(_filename, "neb://", strlen("neb://")))
-
-
-#endif // STREAKS_H
Index: branches/bills_081204/magic/remove/src/streaksastrom.c
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/streaksastrom.c	(revision 20530)
+++ branches/bills_081204/magic/remove/src/streaksastrom.c	(revision 20890)
@@ -1,7 +1,7 @@
-#include "streaks.h"
+#include "streaksremove.h"
 
 // Initialize the astrometry object for current cell
 strkAstrom *
-streakSetAstrometry(strkAstrom *astrom, pmFPA *fpa, pmChip *chip, bool fromCell, psMetadata *md, int numCols, int numRows)
+streakSetAstrometry(strkAstrom *astrom, ippStage stage, pmFPA *fpa, pmChip *chip, bool fromCell, psMetadata *md, int numCols, int numRows)
 {
     if (!astrom) {
@@ -39,11 +39,13 @@
             return false;
         }
-    } else if (md) {
+    } else if (md && (stage == IPP_STAGE_RAW)) {
         // The metadata is the raw header
         // Assumes GPC1
+        //
+        // Shouldn't these lookups be F32 ?
         cell_x0 =  psMetadataLookupS32(&mdok, md, "IMNPIX1");
         if (!mdok) {
             psError(PS_ERR_UNKNOWN, true, "IMNPIX1 for cell is not set.\n");
-            return NULL;
+            // return NULL;
         }
         cell_y0 = psMetadataLookupS32(&mdok, md, "IMNPIX2");
@@ -63,4 +65,5 @@
             return false;
         }
+
     } else {
         // no metadata regular cell
@@ -108,4 +111,6 @@
     outPt->y = (pt->chip->y - astrom->cell_y0) * astrom->yParity;
 
+//    printf("cell: %f %f chip: %f %f\n", outPt->x, outPt->y, pt->chip->x, pt->chip->y);
+
     return true;
 }
@@ -136,3 +141,93 @@
     return true;
 }
+
+void
+cellToChip(PixelPos *chip, strkAstrom *astrom, PixelPos *cell)
+{
+    // TODO: do I need to worry about going from int to float and back again here?
+    chip->x = (cell->x * astrom->xParity) + astrom->cell_x0;
+    chip->y = (cell->y * astrom->yParity) + astrom->cell_y0;
+}
  
+static bool
+readAstrometry(streakFiles *sf)
+{
+    pmHDU *phu = pmFPAviewThisPHU(sf->view, sf->inAstrom->fpa);
+    if (phu) {
+        bool status;
+        char *ctype = psMetadataLookupStr(&status, phu->header, "CTYPE1");
+        if (ctype) {
+            sf->bilevelAstrometry = !strcmp (&ctype[4], "-DIS");
+        }
+    }
+
+    pmHDU *hdu = pmFPAviewThisHDU(sf->view, sf->inAstrom->fpa);
+    PS_ASSERT_PTR_NON_NULL(hdu, 1)
+    PS_ASSERT_PTR_NON_NULL(hdu->header, 1)
+    if (sf->bilevelAstrometry) {
+        // Do we get here for GPC1 ? I don't necessarily have an fpa for the input image
+        if (!pmAstromReadBilevelMosaic(sf->inAstrom->fpa, phu->header)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to read bilevel mosaic astrometry for input FPA.");
+            return false;
+        }
+       if (!pmAstromReadBilevelChip (sf->chip, hdu->header)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to read bilevel chip astrometry for input FPA.");
+            return false;
+        }
+    } else {
+
+        // we use a default FPA pixel scale of 1.0
+        if (!pmAstromReadWCS (sf->inAstrom->fpa, sf->chip, hdu->header, 1.0)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry for input FPA.");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void
+setupAstrometry(streakFiles *sf)
+{
+    bool status;
+    // load astrometry file
+    if (USE_SUPPLIED_ASTROM(sf->stage)) {
+        sf->inAstrom = pmFPAfileDefineFromArgs(&status, sf->config, "PSWARP.ASTROM", "ASTROM");
+    } else {
+        // otherwise get astrometry from pmfile
+        if (!sf->inImage->pmfile) {
+            streaksExit("unexpected null pmFPAfile", PS_EXIT_CONFIG_ERROR);
+        }
+        sf->inAstrom = sf->inImage->pmfile;
+    }
+    sf->view = pmFPAviewAlloc(0);
+
+    if (!pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_BEFORE)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to load input.");
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+
+    while ((sf->chip = pmFPAviewNextChip(sf->view, sf->inAstrom->fpa, 1)))  {
+        if (sf->inAstrom->fpa->chips->n == 1) {
+            // There's only one chip in this FPA and we've got it so break
+            break;
+        }
+        bool status;
+        psString chip_name = psMetadataLookupStr(&status, sf->chip->concepts, "CHIP.NAME");
+        if (!strcmp(chip_name, sf->class_id)) {
+            break;
+        }
+    } 
+    if (!sf->chip) {
+        psError(PS_ERR_UNKNOWN, true, "Failed to find chip with data.");
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    if (!pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_BEFORE)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to load chip");
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    if (!readAstrometry(sf)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to read astrometry");
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+}
Index: branches/bills_081204/magic/remove/src/streaksastrom.h
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/streaksastrom.h	(revision 20530)
+++ branches/bills_081204/magic/remove/src/streaksastrom.h	(revision 20890)
@@ -22,15 +22,9 @@
 
 
-#ifndef notdef
-// There must be some well known type lying around that we
-// can use for this
+// XXX: Since Paul is including ipp headers perhaps we can replace uses of strkPt with psPlane
 typedef struct {
     double  x;
     double  y;
 } strkPt;
-#else
-// TODO: remove this typedef
-typedef psPlane strkPt;
-#endif
 
 extern bool skyToCell(strkPt *, strkAstrom *astrom, double ra, double dec);
Index: branches/bills_081204/magic/remove/src/streakscompare.c
===================================================================
--- branches/bills_081204/magic/remove/src/streakscompare.c	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streakscompare.c	(revision 20890)
@@ -0,0 +1,178 @@
+#include "streaksremove.h"
+
+static pmConfig * parseArguments(int, char **);
+
+int main(int argc, char *argv[])
+{
+    long i;
+    bool status;
+
+    psLibInit(NULL);
+
+    pmConfig *config = parseArguments(argc, argv);
+    if (!config) {
+        streaksExit("failed to parse arguments", PS_EXIT_CONFIG_ERROR);
+    }
+
+    ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE");
+
+    sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true);
+    sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true);
+
+    int ncomponents;
+    if (stage == IPP_STAGE_RAW) {
+        if (file1->nHDU != file2->nHDU) {
+            streaksExit("mages do not have same number of components\n", PS_EXIT_DATA_ERROR);
+        }
+        if (!(psFitsMoveExtNum(file1->fits, 1, true) && psFitsMoveExtNum(file2->fits, 1, true))) {
+            streaksExit("failed to skip primary header of raw file\n", PS_EXIT_DATA_ERROR);
+        }
+        ncomponents = file1->nHDU - 1;
+    } else {
+        ncomponents = 1;
+    }
+
+    int numErrors = 0;
+    for (int component = 0; component < ncomponents; component++) {
+        if (component && !(psFitsMoveExtNum(file1->fits, 1, true) 
+                           && psFitsMoveExtNum(file2->fits, 1, true))) {
+            streaksExit("failed to advance to next extesion\n", PS_EXIT_DATA_ERROR);
+        }
+        readImage(file1, 0, stage);
+        readImage(file2, 0, stage);
+
+        psImage *image1 = file1->image;
+        psImage *image2 = file2->image;
+
+        // TODO: do more sanity checking. For example check that extname's  (if any) match 
+        // check for matching image cubes
+        if (image1 && image2) {
+            if ((image1->numRows != image2->numRows) || (image1->numCols != image2->numCols)) {
+                streaksExit("image sizes do not match\n", PS_EXIT_DATA_ERROR);
+            }
+            int width = image1->numCols;
+            int height = image1->numRows;
+            for (int y = 0; y < height; y++) {
+                for (int x = 0; x < width; x++) {
+                    bool error = false;
+
+                    double value1 = psImageGet(image1, x, y);
+                    double value2 = psImageGet(image1, x, y);
+
+                    if (!isnan(value1) && !isnan(value2)) {
+                        if (value1 != value2) {
+                            error = true;
+                        }
+                    } else {
+                        // if one's a nan they had both better be
+                        error = ! isnan(value1) && isnan(value2);
+                    }
+                    
+                    if (error) {
+                        numErrors++;
+                        fprintf(stderr, "pixel at %5d %5d do not match %f %f\n", x, y, value1, value2);
+                    }
+                }
+            }
+        } else if (! ((image1 == NULL) && (image2 == NULL))) {
+            // if one image is null (video extension) both should be
+            streaksExit("image1 null but not image2!", PS_EXIT_DATA_ERROR);
+        }
+    }
+
+    fprintf(stderr, "%d errors\n", numErrors);
+
+    closeImage(file1);
+    closeImage(file2);
+
+    psFree(config);
+    psLibFinalize();
+
+    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streakscompare");
+
+    //  PAU
+    return 0;
+}
+static void usage(void)
+{
+    fprintf(stderr, "USAGE: streakscompare file1 file2\n");
+
+    exit(2);
+}
+
+static pmConfig *parseArguments(int argc, char **argv)
+{
+    int argnum;
+
+    pmConfig *config = pmConfigRead(&argc, argv, NULL);
+    if (!config) {
+        return NULL;
+    }
+
+    ippStage stage = IPP_STAGE_NONE;
+    if ((argnum = psArgumentGet(argc, argv, "-stage"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        stage = parseStage(argv[argnum]);
+        psMetadataAddS32(config->arguments, PS_LIST_TAIL, "STAGE", 0,
+                "pipeline stage for streak removal", stage);
+        psArgumentRemove(argnum, &argc, argv);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-stage is required\n");
+        usage();
+    }
+
+    // If we are asked to replace inputs with the outputs and input image is in nebulous
+    // we require mask, weight, and tmproot to all be
+    bool nebulousImage = false;
+    if ((argnum = psArgumentGet(argc, argv, "-image1"))) {
+        // for raw and chip level images we use psFits for I/O and ...
+        nebulousImage = IN_NEBULOUS(argv[argnum+1]);
+        if (CHIP_LEVEL_INPUT(stage)) {
+            psArgumentRemove(argnum, &argc, argv);
+            psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT1", 0,
+                    "name of input image", argv[argnum]);
+            psArgumentRemove(argnum, &argc, argv);
+        } else  {
+            // ... for warped images we use pmFPAfiles
+            if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "INPUT1", "-image1", NULL)) { ;
+                psError(PS_ERR_UNKNOWN, false, "failed to process -image");
+                usage();
+            }
+        }
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-image1 is required\n");
+        usage();
+    }
+    if ((argnum = psArgumentGet(argc, argv, "-image2"))) {
+        // for raw and chip level images we use psFits for I/O and ...
+        nebulousImage = IN_NEBULOUS(argv[argnum+1]);
+        if (CHIP_LEVEL_INPUT(stage)) {
+            psArgumentRemove(argnum, &argc, argv);
+            psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT2", 0,
+                    "name of input image", argv[argnum]);
+            psArgumentRemove(argnum, &argc, argv);
+        } else  {
+            // ... for warped images we use pmFPAfiles
+            if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "INPUT2", "-image2", NULL)) { ;
+                psError(PS_ERR_UNKNOWN, false, "failed to process -image");
+                usage();
+            }
+        }
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-image2 is required\n");
+        usage();
+    }
+
+    if (argc != 1) {
+        psString unexpectedArguments = NULL;
+        for (int i=1; i<argc; i++) {
+            psStringAppend(&unexpectedArguments, "%s ", argv[i]);
+        }
+        psError(PS_ERR_UNKNOWN, true, "unexpected arguments: %s", unexpectedArguments);
+        psFree(unexpectedArguments);
+        usage();
+    }
+
+    return config;
+}
+
Index: branches/bills_081204/magic/remove/src/streaksextern.c
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/streaksextern.c	(revision 20530)
+++ branches/bills_081204/magic/remove/src/streaksextern.c	(revision 20890)
@@ -35,4 +35,5 @@
     Line line;
     StreakPixels *pixels = psArrayAllocEmpty (1024);
+    int streaksOnComponent = 0;
     for (i = 0; i != streaks->size; ++i)
     {
@@ -47,6 +48,8 @@
         {
             PixelsFromLine (pixels, &line);
+            streaksOnComponent++;
         }
     }
+    printf("%d streaks on this component\n", streaksOnComponent);
     return pixels;
 }
Index: branches/bills_081204/magic/remove/src/streaksio.c
===================================================================
--- branches/bills_081204/magic/remove/src/streaksio.c	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streaksio.c	(revision 20890)
@@ -0,0 +1,923 @@
+#include "streaksremove.h"
+#include "libgen.h"
+#include "unistd.h"
+
+static nebServer *ourNebServer = NULL;
+
+streakFiles *openFiles(pmConfig *config, bool remove)
+{
+    bool status;
+    streakFiles *sf = psAlloc(sizeof(streakFiles));
+    memset(sf, 0, sizeof(*sf));
+
+    sf->config = config;
+
+    // error checking is done by sFileOpen. If a file can't be opened we just exit
+    ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE");
+
+    sf->stage = stage;
+    sf->extnum = 0;
+
+    sf->class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID");
+
+    sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true);
+    sf->nHDU = sf->inImage->nHDU;
+
+    // don't need to free inputBasename see basename(3)
+    // The names of the temporary and recovery files are taken from the input
+    char *inputBasename = basename(sf->inImage->name);
+    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+
+    if (remove) {
+        sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+    } else {
+        sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true);
+    }
+
+    sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false);
+    if (sf->inMask) {
+        inputBasename = basename(sf->inMask->name);
+        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        if (remove) {
+            sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+        } else {
+            sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true);
+        }
+    }
+    sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false);
+    if (sf->inWeight) {
+        inputBasename = basename(sf->inWeight->name);
+        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        if (remove) {
+            sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+        } else {
+            sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true);
+        }
+    }
+     
+    psElemType tileType;                // Type corresponding to "long"
+    if (sizeof(long) == sizeof(psS64)) {
+        tileType = PS_TYPE_S64;
+    } else if (sizeof(long) == sizeof(psS32)) {
+        tileType = PS_TYPE_S32;
+    } else {
+        psAbort("can't map (long) type to a psLib type");
+    }
+
+    sf->tiles = psVectorAlloc(3, tileType); // Tile sizes
+    if (tileType == PS_TYPE_S64) {
+        sf->tiles->data.S64[0] = 0;
+        sf->tiles->data.S64[1] = 1;
+        sf->tiles->data.S64[2] = 1;
+    } else {
+        sf->tiles->data.S32[0] = 0;
+        sf->tiles->data.S32[1] = 1;
+        sf->tiles->data.S32[2] = 1;
+    }
+
+    sf->transparentStreaks = psMetadataLookupF64(&status, config->arguments, "TRANSPARENT_STREAKS");
+
+    return sf;
+}
+
+void sFileFree(sFile *sfile)
+{
+    psFree(sfile->resolved_name);
+    psFree(sfile->name);
+    psFree(sfile);
+}
+
+
+// getNebServer()
+//
+// it gets created the first time a nebulous file is accessed.
+// if config is passed in we are to create it and exit if not able.
+// if config is null we return NULL indicating that there are no nebulous files in use
+// The purpose of this is to avoid passing extra arguments between all of the functions.
+// but it is probably a bad idea.
+//
+static nebServer *getNebServer(pmConfig *config)
+{
+    if (ourNebServer) {
+        return ourNebServer;
+    }
+
+    if (!config) {
+        return NULL;
+    }
+
+    psString serverURI = getenv("NEB_SERVER");
+
+    // XXX: Note: all of the flags to psError should be true, but I don't think nebclient
+    // uses psError
+    if (!serverURI) {
+        bool status;
+        serverURI = psMetadataLookupStr(&status, config->site, "NEB_SERVER");
+        if (!status) {
+            psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER.");
+            streaksExit("", PS_EXIT_CONFIG_ERROR);
+        }
+    }
+
+    if (!serverURI) {
+        psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI.");
+
+        streaksExit("", PS_EXIT_CONFIG_ERROR);
+    }
+
+    ourNebServer = nebServerAlloc(serverURI);
+    if (!ourNebServer) {
+        psError(PM_ERR_SYS, true, "failed to create a nebServer object from %s.", serverURI);
+        streaksExit("", PS_EXIT_CONFIG_ERROR);
+    }
+
+    return ourNebServer;
+}
+
+static psString
+resolveFilename(pmConfig *config, sFile *sfile, bool create)
+{
+    sfile->inNebulous = IN_NEBULOUS(sfile->name);
+
+    if (sfile->inNebulous) {
+        // make sure we have our neb server connection
+        nebServer *server = getNebServer(config);
+        if (create) {
+            // delete the existing file, since there may be more than one
+            // instance. It will get created below in pmConfigConvertFilename
+            if (nebFind(server, sfile->name)) {
+                nebDelete(server, sfile->name);
+            }
+        }
+    }
+
+    return pmConfigConvertFilename(sfile->name, config, create, create);
+}
+
+sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
+    psString outputFilename, bool required)
+{
+    bool status;
+    sFile *sfile = psAlloc(sizeof(sFile));
+    memset(sfile, 0, sizeof(sFile));
+
+    // We use psFits directly to read the image unless the stage is warp
+    // or diff. In those cases we use the pmFPAfile functions to read the image file
+    // to make managing the astrometry easy.
+    // The reason we don't use pmFPAfile in all cases is that I was having trouble getting
+    // all of the keywords in the raw image files written to the output destreaked files
+
+    if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
+        // stage is warp or diff AND fileSelect eq "INPUT"
+        // get data from pmFPAfile.
+
+        // we need to know what the nebulous and real filenames are so we steal
+        // some code from pmFPAfileDefineFromArgs
+        // XXX: create a pmFPAfile function that does this and use it there
+        psArray *infiles = psMetadataLookupPtr(&status, config->arguments, "INPUT");
+        if (!status) {
+            psError(PS_ERR_PROGRAMMING, false, "INPUT not found in config->arguments");
+            streaksExit("", PS_EXIT_PROG_ERROR);
+        }
+        if (infiles->n != 1) {
+            psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments expencted 1\n",
+                infiles->n, "INPUT");
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        // end of file name lookup code adapted from pmFPAfileDefineFromArgs
+        //
+        sfile->name = psStringCopy(infiles->data[0]);
+        sfile->inNebulous = IN_NEBULOUS(sfile->name);
+
+        // XXX: I should probably be using a different file rule for diffs, but I don't
+        // have an input rule that takes a diff image. 
+        // Since they're skycells, this should be compatible
+        sfile->pmfile = pmFPAfileDefineFromArgs(&status, config, "PPSUB.INPUT", "INPUT");
+        if (!sfile->pmfile) {
+            psError(PS_ERR_IO, false, "Failed to define file for %s", fileSelect);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+
+        pmFPAview *view = pmFPAviewAlloc(0);
+        if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to load input.");
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        psFree(view);
+        sfile->resolved_name = psStringCopy(sfile->pmfile->filename);
+        
+        // copy header from fpu
+        sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header);
+
+        return sfile;
+    }
+
+    // For all other files we use using psFits for i/o
+
+    psString name = psMetadataLookupStr(&status, config->arguments, fileSelect);
+    if (!status || !name) {
+        if (required) {
+            psError(PS_ERR_IO, false, "Failed to lookup name for %s", fileSelect);
+            sFileFree(sfile);
+            streaksExit("", 1);
+        }
+        return NULL;
+    }
+
+    // if outputFilename is not null name it contains the "directory" 
+    // and outputFilename is the basename name of the file (or nebulous key)
+    // and the file is to be opened for writing
+    if (outputFilename) {
+        psStringAppend(&sfile->name, "%s%s", name, outputFilename);
+        sfile->resolved_name = resolveFilename(config, sfile, true);
+        if (!sfile->resolved_name) {
+            psError(PS_ERR_IO, false, "Failed to resolve filename for %s", sfile->name);
+            sFileFree(sfile);
+            streaksExit("", 1);
+        }
+        sfile->fits = psFitsOpen(sfile->resolved_name, "w");
+        if (sfile->fits) {
+            sfile->fits->options = psFitsOptionsAlloc();
+        }
+    } else {
+        sfile->name = psStringCopy(name);
+        sfile->resolved_name = resolveFilename(config, sfile, false);
+        if (!sfile->resolved_name) {
+            psError(PS_ERR_IO, false, "Failed to resolve name for %s", sfile->name);
+            sFileFree(sfile);
+            streaksExit("", 1);
+        }
+        sfile->fits = psFitsOpen(sfile->resolved_name, "r");
+        if (sfile->fits) {
+            sfile->nHDU = psFitsGetSize(sfile->fits);
+        }
+    }
+
+    if (!sfile->fits) {
+        psError(PS_ERR_IO, false, "failed to open fits file %s for %s",
+                    sfile->resolved_name, outputFilename ? "writing" : "reading");
+        sFileFree(sfile);
+        streaksExit("", 1);
+    }
+
+    return sfile;
+}
+
+
+static bool
+moveExt(sFile *sfile, int extnum)
+{
+    bool status = psFitsMoveExtNum(sfile->fits, extnum, false);
+    if (!status) {
+        psError(PS_ERR_IO, false, 
+            "failed to move to extension %d for %s", extnum, sfile->resolved_name);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    return true;
+}
+
+bool
+streakFilesNextExtension(streakFiles *sf)
+{
+    // unless stage is raw or chip when we get here we're done
+    if (sf->stage != IPP_STAGE_RAW) {
+        if (sf->view) {
+            sf->view->readout = -1;
+            pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
+            sf->view->cell = -1;
+            pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
+            sf->view->chip = -1;
+            pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
+        }
+        return false;
+    }
+
+    ++sf->extnum;
+    if (sf->nHDU == 1) {
+        // return true the first time through
+        return (sf->extnum == 1);
+    }
+    if (sf->extnum < sf->nHDU) {
+        moveExt(sf->inImage, sf->extnum);
+        if (sf->inMask) {
+            moveExt(sf->inMask, sf->extnum);
+        }
+        if (sf->inWeight) {
+            moveExt(sf->inWeight, sf->extnum);
+        }
+        return true;
+    } else {
+        return false;
+    }
+}
+
+void
+copyPHU(streakFiles *sfiles, bool remove)
+{
+    psAssert(sfiles->stage == IPP_STAGE_RAW, "copyPHU should only be used for raw stage");
+
+    psMetadata *imageHeader = psFitsReadHeader(NULL, sfiles->inImage->fits);
+    if (!imageHeader) {
+        psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
+            sfiles->inImage->resolved_name);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+
+    // TODO: add keyword indicating that streaks have been removed
+    if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
+        psError(PS_ERR_IO, false, "failed to write primary header to %s",
+            sfiles->outImage->resolved_name);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    // TODO: add keyword indicating that this is the recovery image
+    if (remove && sfiles->recImage && !psFitsWriteBlank(sfiles->recImage->fits, imageHeader, NULL)) {
+        psError(PS_ERR_IO, false, "failed to write primary header to %s",
+            sfiles->recImage->resolved_name);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    psFree(imageHeader);
+
+    sFile *inMask = sfiles->inMask;
+    if (inMask) {
+        psMetadata *maskHeader = psFitsReadHeader(NULL, inMask->fits);
+        if (!maskHeader) {
+            psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
+                sfiles->inMask->resolved_name);
+            streaksExit("", 1);
+        }
+        // TODO: add keyword indicating that streaks have been removed
+        if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
+            psError(PS_ERR_IO, false, "failed to write primary header to %s",
+                sfiles->outMask->resolved_name);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        // TODO: add keyword indicating that this is the recovery image
+        if (remove && sfiles->recMask && !psFitsWriteBlank(sfiles->recMask->fits, maskHeader, NULL)) {
+            psError(PS_ERR_IO, false, "failed to write primary header to %s",
+                sfiles->recMask->resolved_name);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        psFree(maskHeader);
+    }
+    sFile *inWeight = sfiles->inWeight;
+    if (inWeight) {
+        psMetadata *weightHeader = psFitsReadHeader(NULL, inWeight->fits);
+        if (!weightHeader) {
+            psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
+                sfiles->inWeight->resolved_name);
+            streaksExit("", 1);
+        }
+        // TODO: add keyword indicating that streaks have been removed
+        if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
+            psError(PS_ERR_IO, false, "failed to write primary header to %s",
+                sfiles->outWeight->resolved_name);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        // TODO: add keyword indicating that this is a recovery image
+        if (remove && sfiles->recWeight && !psFitsWriteBlank(sfiles->recWeight->fits, weightHeader, NULL)) {
+            psError(PS_ERR_IO, false, "failed to write primary header to %s",
+                sfiles->recWeight->resolved_name);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        psFree(weightHeader);
+    }
+}
+
+// Determine whether the current header for this file is an image or not
+// Find a cleaner way to do this
+static bool
+isImage(sFile *in)
+{
+    bool status;
+    psString exttype = psMetadataLookupStr(&status, in->header, "EXTTYPE");
+    if (exttype && !strcmp(exttype, "IMAGE")) {
+        return true;
+    }
+    // examine current header and determine if it is an image
+    psString xtension = psMetadataLookupStr(&status, in->header, "XTENSION");
+    if (xtension) {
+        if (!strcmp(xtension, "IMAGE")) {
+            return true;
+        } else if (!strcmp(xtension, "BINTABLE")) {
+            psString ztension = psMetadataLookupStr(&status, in->header, "ZTENSION");
+            if (ztension && !strcmp(ztension, "IMAGE")) {
+                return true;
+            }
+        }
+    } else if (psMetadataLookupBool(&status, in->header, "ZIMAGE")) {
+        return true;
+    } else if (in->nHDU == 1) {
+        // no extensions in the file, can just return true? For now require 
+        // at least one dimension
+        int naxis =  psMetadataLookupS32(&status, in->header, "NAXIS");
+        return naxis > 0;
+    }
+    return false;
+}
+
+void
+readImageFrom_pmFile(streakFiles *sf)
+{
+    // XXX: This function assumes that it is only used for a single
+    // chip single cell FPA (i.e. a skycell)
+    pmFPAview *view = sf->view;
+    assert(view->chip == 0);
+    view->cell = 0;
+    sf->cell =  pmFPAviewThisCell(view, sf->inImage->pmfile->fpa);
+
+    if (!sf->cell) {
+        streaksExit("no cells found in chip", PS_EXIT_PROG_ERROR);
+    }
+    if (sf->cell->readouts->n != 1) {
+        psError(PS_ERR_PROGRAMMING, true, "unexpected number of readouts found: %ld", sf->cell->readouts->n);
+        streaksExit("", PS_EXIT_PROG_ERROR);
+    }
+    view->readout = 0;
+    pmReadout *readout = pmFPAviewThisReadout(view, sf->inImage->pmfile->fpa);
+    if (!readout) {
+        streaksExit("readout 0 not found", PS_EXIT_PROG_ERROR);
+    }
+
+    sf->inImage->image = (psImage*) psMemIncrRefCounter(readout->image);
+    sf->inImage->numCols = readout->image->numCols;
+    sf->inImage->numRows = readout->image->numRows;
+}
+
+void
+setDataExtent(ippStage stage, sFile *in)
+{
+    if (stage == IPP_STAGE_RAW) {
+        psString datasec = psMetadataLookupStr(NULL, in->header, "DATASEC");
+        if (!datasec) {
+            psError(PS_ERR_IO, false, "failed to find DATASEC in header");
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        int xmin, xmax, ymin, ymax;
+        sscanf(datasec, "[%d:%d,%d:%d]", &xmin, &xmax, &ymin, &ymax);
+        in->numCols = xmax - xmin + 1;
+        in->numRows = ymax - ymin + 1;
+    } else {
+        in->numCols = in->image->numCols;
+        in->numRows = in->image->numRows;
+    }
+}
+
+void
+readImage(sFile *in, int extnum, ippStage stage)
+{
+    psRegion region = {0, 0, 0, 0};
+
+    if (in->header) psFree(in->header);
+    if (in->image) {
+        psFree(in->image);
+        in->image = NULL;
+    }
+    if (in->imagecube) {
+        psFree(in->imagecube);
+        in->imagecube = NULL;
+    }
+
+    in->header = psFitsReadHeader(NULL, in->fits);
+    if (!in->header) {
+        psError(PS_ERR_IO, false, "failed to read header from %s extnum: %d", 
+            in->resolved_name, extnum);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+
+    bool status;
+    in->extname = psMetadataLookupStr(&status, in->header, "EXTNAME");
+
+    if (!isImage(in)) {
+        return;
+    }
+
+    in->numZPlanes = psMetadataLookupS32(NULL, in->header, "NAXIS3");
+
+    if (in->numZPlanes == 0) {
+        in->image = psFitsReadImage(in->fits, region, 0);
+        if (!in->image) {
+            psError(PS_ERR_IO, false, "failed to read image from %s extnum: %d", 
+                in->resolved_name, extnum);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+    }  else {
+        if (stage != IPP_STAGE_RAW) {
+            psError(PS_ERR_PROGRAMMING, true, "unexpected image cube found in non-raw image");
+            streaksExit("", PS_EXIT_PROG_ERROR);
+        }
+        in->imagecube = psFitsReadImageCube(in->fits, region);
+        if (!in->imagecube) {
+            psError(PS_ERR_IO, false, "failed to read image cube from %s extnum: %d", 
+                in->resolved_name, extnum);
+            streaksExit("", PS_EXIT_DATA_ERROR);
+        }
+        psImage *image = (psImage *) (in->imagecube->data[0]);
+    }
+    setDataExtent(stage, in);
+}
+
+static void
+setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero)
+{
+    if (!sfile) {
+        return;
+    }
+    if (sfile->fits->options) {
+        psFree(sfile->fits->options);
+    }
+    sfile->fits->options = psFitsOptionsAlloc();
+    sfile->fits->options->scaling = PS_FITS_SCALE_MANUAL;
+    sfile->fits->options->bitpix = bitpix;
+    sfile->fits->options->bscale = bscale;
+    sfile->fits->options->bzero = bzero;
+}
+
+void
+copyFitsOptions(sFile *out, sFile *rec, sFile *in)
+{
+    // Get current BITPIX, BSCALE, BZERO, EXTNAME
+    // Probably not necessary to look the numerical values up in this
+    // way, but guards against changes to psLib and cfitsio FITS
+    // handling.
+    psMetadataItem *bitpixItem = psMetadataLookup(in->header, "BITPIX");
+    psAssert(bitpixItem, "Every FITS image should have BITPIX");
+    int bitpix = psMetadataItemParseS32(bitpixItem);
+    psMetadataItem *bscaleItem = psMetadataLookup(in->header, "BSCALE");
+
+    float bscale;
+    if (!bscaleItem) {
+        psWarning("BSCALE isn't set; defaulting to unity");
+        bscale = 1.0;
+    } else {
+        bscale = psMetadataItemParseF32(bscaleItem);
+    }
+    psMetadataItem *bzeroItem = psMetadataLookup(in->header, "BZERO");
+    float bzero;
+    if (!bzeroItem) {
+        psWarning("BZERO isn't set; defaulting to zero");
+        bzero = 0.0;
+    } else {
+        bzero = psMetadataItemParseF32(bzeroItem);
+    }
+
+#ifdef STREAKS_COMPRESS_OUTPUT
+    setFitsOptions(out, bitpix, bscale, bzero);
+    setFitsOptions(rec, bitpix, bscale, bzero);
+#endif
+}
+
+
+void
+copyTable(sFile *out, sFile *in, int extnum)
+{
+    bool mdok;
+    psString xtension = psMetadataLookupStr(&mdok, in->header, "XTENSION");
+    psString extname = psMetadataLookupStr(&mdok, in->header, "EXTNAME");
+
+    if (!xtension) {
+        psWarning("extnum %d has no image and xtension is not defined in %s",
+            extnum, in->resolved_name);
+        // XXX abort?
+        return;
+    } 
+    if (!extname) {
+        psWarning("extnum %d has no image and extname not defined in %s",
+            extnum, in->resolved_name);
+        // XXX abort?
+        return;
+    }
+    psArray *table = psFitsReadTable(in->fits);
+    if (!table) {
+        psError(PS_ERR_UNKNOWN, false, "failed to read table in extension %d from in->resolved name", extnum);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+
+    if (!psFitsWriteTable(out->fits, out->header, table, extname)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to copy table in extension %d", extnum);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+}
+
+static void
+createNewImage(sFile *out, sFile *in, int extnum, double initValue)
+{
+    out->image = psImageRecycle(out->image, in->image->numCols, in->image->numRows, in->image->type.type);
+    if (!out->image) {
+        psError(PS_ERR_UNKNOWN, false, "failed to allocate image for extnsion %d", extnum);
+        streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+    }
+    psImageInit(out->image, initValue);
+}
+
+void
+setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll)
+{
+    if (!exciseAll) {
+        // set output image to be the input image
+        out->image = (psImage*) psMemIncrRefCounter(in->image);
+        if (recoveryOut) {
+            // create a recovery image initialized to NAN
+            createNewImage(recoveryOut , in, extnum, NAN);
+        }
+    } else {
+        // Excising all pixels in the input
+        // set the output to an image all NANs
+        createNewImage(out, in, extnum, NAN);
+        if (recoveryOut) {
+            // save the whole input as the recovery image
+            recoveryOut->image = (psImage*) psMemIncrRefCounter(in->image);
+        }
+    }
+}
+
+void
+writeImage(sFile *sfile, psString extname, int extnum)
+{
+    if (!sfile || !sfile->image) {
+        return;
+    }
+    if (!psFitsWriteImage(sfile->fits, sfile->header, sfile->image, 0, extname)) {
+        psError(PS_ERR_IO, false, "failed to write image to %s extnum: %d", 
+            sfile->resolved_name, extnum);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    psFree(sfile->image);
+    sfile->image = NULL;
+    psFree(sfile->header);
+    sfile->header = NULL;
+}
+
+void
+writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum)
+{
+    if (!imagecube) {
+        return;
+    }
+    if (!psFitsWriteImageCube(sfile->fits, sfile->header, imagecube, extname)) {
+        psError(PS_ERR_IO, false, "failed to write image to %s extnum: %d", 
+            sfile->resolved_name, extnum);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    psFree(sfile->header);
+    sfile->header = NULL;
+}
+
+#ifdef notdef
+void
+writeImages(streakFiles *sf, bool exciseImageCube)
+{
+    psString extname = NULL;
+    if (sf->nHDU > 1) {
+        bool mdok;
+        extname = psMetadataLookupStr(&mdok, sf->inImage->header, "EXTNAME");
+    }
+    if (sf->inImage->numZPlanes == 0)  {
+        // note exciseing complete images is handled in readAndCopyToOutput
+        writeImage(sf->outImage, extname, sf->extnum);
+        writeImage(sf->recImage, extname, sf->extnum);
+    } else {
+        // we have an image cube
+        double initValue;
+        if (exciseImageCube) {
+            // copy the entire input image to the recovery image
+            writeImageCube(sf->recImage, sf->inImage->imagecube, extname, sf->extnum);
+            initValue = NAN;
+        } else {
+            // otherwise write it to the output 
+            writeImageCube(sf->outImage, sf->inImage->imagecube, extname, sf->extnum);
+            initValue = 0;
+        }
+
+        // borrow one of the images from the imagecube and set it to init value
+        psImage *image = psArrayGet (sf->inImage->imagecube, 0);
+        psMemIncrRefCounter(image);
+        psImageInit(image, initValue);
+        if (exciseImageCube) {
+            sf->outImage->image = image;
+            writeImage(sf->outImage, extname, sf->extnum);
+        } else {
+            // write zero valued image to reccovery
+            if (sf->recImage) {
+                sf->recImage->image = image;
+                writeImage(sf->recImage, extname, sf->extnum);
+            }
+        }
+        // Assumption: there are no mask and weight images for video cells
+        return;
+    }
+    if (sf->outMask) {
+        writeImage(sf->outMask, extname, sf->extnum);
+        writeImage(sf->recMask, extname, sf->extnum);
+    }
+    if (sf->outWeight) {
+        writeImage(sf->outWeight, extname, sf->extnum);
+        writeImage(sf->recWeight, extname, sf->extnum);
+    }
+}
+#endif
+
+void
+closeImage(sFile *sfile)
+{
+    if (!sfile) {
+        return;
+    }
+    if (sfile->fits && !psFitsClose(sfile->fits)) {
+        psError(PS_ERR_IO, false, "failed to close image to %s", 
+            sfile->resolved_name);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+    }
+    psFree(sfile->header);
+    psFree(sfile->image);
+    psFree(sfile->imagecube);
+    psFree(sfile->name);
+    psFree(sfile->resolved_name);
+    psFree(sfile);
+}
+
+void
+closeImages(streakFiles *sf)
+{
+    closeImage(sf->inImage);
+    closeImage(sf->outImage);
+    closeImage(sf->recImage);
+    if (sf->inMask) {
+        closeImage(sf->inMask);
+        closeImage(sf->outMask);
+        closeImage(sf->recMask);
+    }
+    if (sf->inWeight) {
+        closeImage(sf->inWeight);
+        closeImage(sf->outWeight);
+        closeImage(sf->recWeight);
+    }
+}
+
+bool
+replicate(sFile *sfile, void *xattr)
+{
+    if (!sfile->inNebulous) {
+        return true;
+    }
+    nebServer *server = getNebServer(NULL);
+
+    // for now just set "user.copies" to 2
+    if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
+        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+        return false;
+    }
+    if (!nebReplicate(server, sfile->name, NULL, NULL)) {
+        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+        return false;
+    }
+    return true;
+}
+
+// for any of the outputs that are stored in Nebulous set their extended attributes
+// and replicate the files
+
+static bool
+replicateOutputs(streakFiles *sfiles)
+{
+    bool status = false;
+
+    // XXX: TODO: need a nebGetXatrr function, but there isn't one
+    // another option would be to take the number of copies to be
+    // created as an option. That way the system could decide
+    // whether to replicate anything other than raw Image files
+    void *xattr = NULL;
+
+    if (!replicate(sfiles->outImage, xattr)) {
+        psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+        return false;
+    }
+
+#ifdef notyet
+    // XXX: don't replicate mask and weight images until we can look up
+    // the input's xattr. There may be a perl program that can getXattr
+    if (sfiles->outMask) {
+        // get xattr from input to see if we need to replicate
+        if (!replicate(sfiles->outMask, xattr)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+    if (sfiles->outWeight) {
+        // get xattr from input to see if we need to replicate
+        if (!replicate(sfiles->outWeight, xattr)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+#endif
+
+//      replicate the recovery images (if in nebulous)
+//      perhaps whether we do that or not should be configurable.
+//      Sounds like we need a recipe
+
+    return true;
+}
+
+static bool
+swapOutputToInput(sFile *in, sFile *out)
+{
+
+    if (in->inNebulous) {
+        nebServer *server = getNebServer(NULL);
+
+        if (!nebSwap(server, in->name, out->name)) {
+            psError(PM_ERR_SYS, true, "failed to swap files for: %s.",
+                in->name, nebErr(server));
+            return false;
+        }
+    } else {
+        psString command = NULL;
+
+        // regular files. use mv creating a backup
+        // NOTE: -b is a gnu option and may not always be available
+        psStringAppend(&command, "mv -b --suffix=.bak %s %s", out->resolved_name, in->resolved_name);
+        int status = system(command);
+        if (status != 0) {
+            psError(PM_ERR_SYS, true, "%s failed with status %d.",
+                command, status);
+            psFree(command);
+            return false;
+        }
+        psFree(command);
+        // XXX: shouldn't I be doing mv in->resolved_name.bak out->resolved_name
+        // to complete the swap if !remove?
+    }
+
+    return true;
+}
+static bool
+swapOutputsToInputs(streakFiles *sfiles)
+{
+    if (sfiles->inMask) {
+        if (!swapOutputToInput(sfiles->inMask, sfiles->outMask)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for Mask.");
+            return false;
+        }
+    }
+
+    if (sfiles->inWeight) {
+        if (!swapOutputToInput(sfiles->inWeight, sfiles->outWeight)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for Weight.");
+            return false;
+        }
+    }
+
+    if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
+        psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
+        return false;
+    }
+
+    return true;
+}
+
+static bool
+deleteFile(sFile *sfile)
+{
+
+    if (sfile->inNebulous) {
+        nebServer *server = getNebServer(NULL);
+        if (!nebDelete(server, sfile->name)) {
+            psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfile->name,
+                nebErr(server));
+            return false;
+        }
+    } else {
+        if (unlink(sfile->resolved_name)) {
+            psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfile->resolved_name,
+                strerror(errno));
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool
+deleteTemps(streakFiles *sfiles)
+{
+    if (sfiles->outMask) {
+        if (!deleteFile(sfiles->outMask)) {
+            psError(PM_ERR_SYS, false, "failed to delete Mask.");
+            return false;
+        }
+    }
+
+    if (sfiles->outWeight) {
+        if (!deleteFile(sfiles->outWeight)) {
+            psError(PM_ERR_SYS, false, "failed to delete Weight.");
+            return false;
+        }
+    }
+
+    if (!deleteFile(sfiles->outImage)) {
+        psError(PM_ERR_SYS, false, "failed to delete Image.");
+        return false;
+    }
+
+    return true;
+}
+
Index: branches/bills_081204/magic/remove/src/streaksio.h
===================================================================
--- branches/bills_081204/magic/remove/src/streaksio.h	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streaksio.h	(revision 20890)
@@ -0,0 +1,27 @@
+#ifndef STREAKS_IO_H
+#define STREAKS_IO_H 1
+
+streakFiles *openFiles(pmConfig *config, bool remove);
+
+sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
+    psString outputFilename, bool required);
+
+void sFileFree(sFile *);
+
+void closeImages(streakFiles *sfiles);
+void closeImage(sFile *sfile);
+
+void readImage(sFile *sfile, int extnum, ippStage stage);
+void copyPHU(streakFiles *sfiles, bool remove);
+void copyTable(sFile *out, sFile *in, int extnum);
+void copyFitsOptions(sFile *out, sFile *rec, sFile *in);
+void setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll);
+
+void writeImage(sFile *sfile, psString extname, int extnum);
+void writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum);
+bool replicate(sFile *sfile, void *xattr);
+void readImageFrom_pmFile(streakFiles *sf);
+
+bool streakFilesNextExtension(streakFiles *sf);
+
+#endif // STREAKS_IO_H
Index: branches/bills_081204/magic/remove/src/streaksremove.c
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/streaksremove.c	(revision 20530)
+++ branches/bills_081204/magic/remove/src/streaksremove.c	(revision 20890)
@@ -1,446 +1,215 @@
-#include "streaks.h"
-#include "streaksextern.h"
-#include "libgen.h"
-#include "unistd.h"
-
-extern bool  sFileLock(sFile * sfile);
-extern bool  sFileUnlock(sFile * sfile);
-extern bool  sFileReplace(sFile *dest, sFile *src);
-
-static nebServer *ourNebServer = NULL;
-
-// for clarity in this program we don't propagate errors up the stack
-// we just bail out
-void streaksremoveExit(psString str, int exitCode) {
-    psErrorStackPrint(stderr, str);
-    exit(exitCode);
-}
-
-static void sFileFree(sFile *sfile)
-{
-    psFree(sfile->resolved_name);
-    psFree(sfile->name);
-    psFree(sfile);
-}
-
-// getNebServer()
-//
-// it gets created the first time a nebulous file is accessed.
-// if config is passed in we are to create it and exit if not able.
-// if config is null we return NULL indicating that there are no nebulous files in use
-// The purpose of this is to avoid passing extra arguments between all of the functions.
-// but it is probably a bad idea.
-//
-static nebServer *getNebServer(pmConfig *config)
-{
-    if (ourNebServer) {
-        return ourNebServer;
-    }
-
+#include "streaksremove.h"
+
+static pmConfig *parseArguments(int argc, char **argv);
+static bool readAndCopyToOutput(streakFiles *sf, bool exciseAll);
+static StreakPixels * getStreakPixels(streakFiles *sfiles, Streaks *streaks);
+static void exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue);
+static bool warpedPixel(streakFiles *sfiles, PixelPos *cellCoord);
+static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue);
+static void writeImages(streakFiles *sf, bool exciseImageCube);
+static bool replicateOutputs(streakFiles *sfiles);
+
+int
+main(int argc, char *argv[])
+{
+    bool status;
+
+    psLibInit(NULL);
+
+    pmConfig *config = parseArguments(argc, argv);
     if (!config) {
-        return NULL;
-    }
-
-    psString serverURI = getenv("NEB_SERVER");
-
-    // XXX: Note: all of the flags to psError should be true, but I don't think nebclient
-    // uses psError
-    if (!serverURI) {
-        bool status;
-        serverURI = psMetadataLookupStr(&status, config->site, "NEB_SERVER");
-        if (!status) {
-            psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER.");
-            streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
-        }
-    }
-
-    if (!serverURI) {
-        psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI.");
-
-        streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
-    }
-
-    ourNebServer = nebServerAlloc(serverURI);
-    if (!ourNebServer) {
-        psError(PM_ERR_SYS, true, "failed to create a nebServer object from %s.", serverURI);
-        streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
-    }
-
-    return ourNebServer;
-}
-
-psString
-resolveFilename(pmConfig *config, sFile *sfile, bool create)
-{
-    sfile->inNebulous = IN_NEBULOUS(sfile->name);
-
-    if (sfile->inNebulous) {
-        // make sure we have our neb server connection
-        nebServer *server = getNebServer(config);
-        if (create) {
-            // delete the existing file, since there may be more than one
-            // instance. It will get created below in pmConfigConvertFilename
-            if (nebFind(server, sfile->name)) {
-                nebDelete(server, sfile->name);
+        psError(PS_ERR_UNKNOWN, false, "failed to parse arguments\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+    double maskStreak = (double) psMetadataLookupU8(&status, masks, "STREAK");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS");
+
+    Streaks *streaks = readStreaksFile(streaksFileName);
+    if (!streaks) {
+        psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName);
+        exit (PS_EXIT_PROG_ERROR);
+    }
+
+    streakFiles *sfiles = openFiles(config, true);
+    setupAstrometry(sfiles);
+
+    bool exciseAll = false;
+    // --keepnonwarped is a test and debug mode
+    bool keepNonWarpedPixels = psMetadataLookupBool(&status, config->arguments, "KEEP_NON_WARPED");
+
+    // we need to check for non warped pixels unless we've been asked not to or the stage is diff
+    // (By definition pixels in diff images were included in a diff)
+    bool checkNonWarpedPixels = ! (keepNonWarpedPixels || (sfiles->stage == IPP_STAGE_DIFF) );
+
+    if (checkNonWarpedPixels ) {
+        // From ICD:
+        // In the raw and detrended images, the pixels which were not
+        // included in any of the streak-processed warps must also be masked.
+        // Note that the warp and difference skycells are only generated if more
+        // than a small fraction of the pixels are lit by the input image.
+        // if no skycells are provided sfiles->exciseAll is set to true
+
+        if (! computeWarpedPixels(sfiles) ) {
+            // we have no choice to excise all pixels
+            exciseAll = true;
+        }
+    }
+    
+    if (sfiles->stage == IPP_STAGE_RAW) {
+        // copy PHU to output files
+        copyPHU(sfiles, true);
+
+        // advance to the first image extension
+        if (!streakFilesNextExtension(sfiles)) {
+            psErrorStackPrint(stderr, "failed to advance to first extension of input images");
+            exit (PS_EXIT_PROG_ERROR);
+        }
+    }
+
+    // Iterate through each component of the input (only one except for raw stage)
+    do {
+        bool exciseImageCube = false;
+
+        // read the images and copy the data from the inputs to the outputs
+        // This also sets up the astrometry and initializes the recovery image(s)
+
+        if (!readAndCopyToOutput(sfiles, exciseAll)) {
+            // false return value indicates that this extension is not an image and
+            // the contents has already been copied to the output file.
+            // we don't need to process this extesion for streaks.
+            continue;
+        }
+
+        if (!exciseAll) {
+            // Identify pixels to mask because of streaks
+
+            StreakPixels *pixels = getStreakPixels(sfiles, streaks);
+
+            // XXX: 
+            // 
+            // PFS: Need to get mask weight for PM_MASK_DETECTOR.  Is this stored
+            // in a config somewhere?  Not sure how to properly set the maskValue.
+
+
+            if (checkNonWarpedPixels) {
+                exciseNonWarpedPixels(sfiles, maskStreak);
             }
-        }
-    }
-
-    return pmConfigConvertFilename(sfile->name, config, create, create);
-}
-
-sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
-    psString outputFilename, bool required)
-{
-    bool status;
-    sFile *sfile = psAlloc(sizeof(sFile));
-    memset(sfile, 0, sizeof(sFile));
-
-    // We use psFits directly to read the image unless the stage is warp
-    // or diff. In those cases we use the pmFPAfile functions to read the image file
-    // to make managing the astrometry easy.
-    // The reason we don't use pmFPAfile in all cases is that I was having trouble getting
-    // all of the keywords in the raw image files written to the output destreaked files
-
-    if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
-        // stage is warp or diff AND fileSelect eq "INPUT"
-        // get data from pmFPAfile.
-
-        // we need to know what the nebulous and real filenames are so we steal
-        // some code from pmFPAfileDefineFromArgs
-        // XXX: create a pmFPAfile function that does this and use it there
-        psArray *infiles = psMetadataLookupPtr(&status, config->arguments, "INPUT");
-        if (!status) {
-            psError(PS_ERR_PROGRAMMING, false, "INPUT not found in config->arguments");
-            streaksremoveExit("", PS_EXIT_PROG_ERROR);
-        }
-        if (infiles->n != 1) {
-            psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments expencted 1\n",
-                infiles->n, "INPUT");
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        // end of file name lookup code adapted from pmFPAfileDefineFromArgs
-        //
-        sfile->name = psStringCopy(infiles->data[0]);
-        sfile->inNebulous = IN_NEBULOUS(sfile->name);
-
-        // XXX: I should probably be using a different file rule for the diff, but I don't
-        // have an input rule that takes a diff image. This should be compatible
-        sfile->pmfile = pmFPAfileDefineFromArgs(&status, config, "PPSUB.INPUT", "INPUT");
-        if (!sfile->pmfile) {
-            psError(PS_ERR_IO, false, "Failed to defile file for name for %s", fileSelect);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-
-        pmFPAview *view = pmFPAviewAlloc(0);
-        if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to load input.");
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        psFree(view);
-        sfile->resolved_name = psStringCopy(sfile->pmfile->filename);
-        
-        // copy header from fpu
-        sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header);
-
-        return sfile;
-    }
-
-    // For all other files we use using psFits for i/o
-
-    psString name = psMetadataLookupStr(&status, config->arguments, fileSelect);
-    if (!status || !name) {
-        if (required) {
-            psError(PS_ERR_IO, false, "Failed to lookup name for %s", fileSelect);
-            sFileFree(sfile);
-            streaksremoveExit("", 1);
-        }
-        return NULL;
-    }
-
-    // if outputFilename is not null name it contains the "directory" 
-    // and outputFilename is the basename name of the file (or nebulous key)
-    // and the file is to be opened for writing
-    if (outputFilename) {
-        psStringAppend(&sfile->name, "%s%s", name, outputFilename);
-        sfile->resolved_name = resolveFilename(config, sfile, true);
-        if (!sfile->resolved_name) {
-            psError(PS_ERR_IO, false, "Failed to create file %s", sfile->name);
-            sFileFree(sfile);
-            streaksremoveExit("", 1);
-        }
-        sfile->fits = psFitsOpen(sfile->resolved_name, "w");
-        sfile->fits->options = psFitsOptionsAlloc();
-    } else {
-        sfile->name = psStringCopy(name);
-        sfile->resolved_name = resolveFilename(config, sfile, false);
-        if (!sfile->resolved_name) {
-            psError(PS_ERR_IO, false, "Failed to resolve name for %s", sfile->name);
-            sFileFree(sfile);
-            streaksremoveExit("", 1);
-        }
-        sfile->fits = psFitsOpen(sfile->resolved_name, "r");
-        if (sfile->fits) {
-            sfile->nHDU = psFitsGetSize(sfile->fits);
-        }
-    }
-
-    if (!sfile->fits) {
-        psError(PS_ERR_IO, false, "failed to open fits file %s for %s",
-                    sfile->resolved_name, outputFilename ? "writing" : "reading");
-        sFileFree(sfile);
-        streaksremoveExit("", 1);
-    }
-
-    return sfile;
-}
-
-
-bool
-astrometry_read(streakFiles *sf)
-{
-    pmHDU *phu = pmFPAviewThisPHU(sf->view, sf->inAstrom->fpa);
-    if (phu) {
-        char *ctype = psMetadataLookupStr(NULL, phu->header, "CTYPE1");
-        if (ctype) {
-            sf->bilevelAstrometry = !strcmp (&ctype[4], "-DIS");
-        }
-    }
-
-    if (sf->bilevelAstrometry) {
-        // Do we get here for GPC1 ? I don't necessarily have an fpa for the input image
-        if (!pmAstromReadBilevelMosaic(sf->inAstrom->fpa, phu->header)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to read bilevel mosaic astrometry for input FPA.");
-            return false;
-        }
-    } else {
-        pmHDU *hdu = pmFPAviewThisHDU(sf->view, sf->inAstrom->fpa);
-        PS_ASSERT_PTR_NON_NULL(hdu, 1)
-        PS_ASSERT_PTR_NON_NULL(hdu->header, 1)
-
-        // we use a default FPA pixel scale of 1.0
-        if (!pmAstromReadWCS (sf->inAstrom->fpa, sf->chip, hdu->header, 1.0)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry for input FPA.");
-            return false;
-        }
-    }
-
-    return true;
-}
-
-static void
-setupAstromFromFPA(streakFiles *sf)
-{
-    sf->view = pmFPAviewAlloc(0);
-
-    if (!pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_BEFORE)) {
-        psError(PS_ERR_UNKNOWN, false, "Failed to load input.");
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-
-    while ((sf->chip = pmFPAviewNextChip(sf->view, sf->inAstrom->fpa, 1)))  {
-        if (sf->inAstrom->fpa->chips->n == 1) {
-            // There's only one chip in this FPA and we've got it so break
-            break;
-        }
-        bool status;
-        psString chip_name = psMetadataLookupStr(&status, sf->chip->concepts, "CHIP.NAME");
-        if (!strcmp(chip_name, sf->class_id)) {
-            break;
-        }
-    } 
-    if (!sf->chip) {
-        psError(PS_ERR_UNKNOWN, true, "Failed to find chip with data.");
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    if (!pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_BEFORE)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to load chip");
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-
-    // read in the astrometry
-    astrometry_read(sf);
-}
-
-streakFiles *openFiles(pmConfig *config)
-{
-    bool status;
-    streakFiles *sf = psAlloc(sizeof(streakFiles));
-    memset(sf, 0, sizeof(*sf));
-
-    sf->config = config;
-
-    // error checking is done by sFileOpen. If a file can't be opened we just exit
-    ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE");
-
-    sf->stage = stage;
-    sf->extnum = 0;
-
-    sf->class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID");
-
-    sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true);
-    sf->nHDU = sf->inImage->nHDU;
-
-    // don't need to free inputBasename see basename(3)
-    // The names of the temporary and recovery files are taken from the input
-    char *inputBasename = basename(sf->inImage->name);
-    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
-    sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
-
-    sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false);
-    if (sf->inMask) {
-        inputBasename = basename(sf->inMask->name);
-        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
-        sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
-    }
-    sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false);
-    if (sf->inWeight) {
-        inputBasename = basename(sf->inWeight->name);
-        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
-        sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
-    }
-
-    // load astrometry file
-    if (USE_SUPPLIED_ASTROM(sf->stage)) {
-        sf->inAstrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
-    } else {
-        // otherwise get astrometry from pmfile
-        if (!sf->inImage->pmfile) {
-            streaksremoveExit("unexpected null pmFPAfile", PS_EXIT_CONFIG_ERROR);
-        }
-        sf->inAstrom = sf->inImage->pmfile;
-    }
-    setupAstromFromFPA(sf);
-    if (CHIP_LEVEL_INPUT(sf->stage)) {
-        computeWarpedPixels(sf);
-    }
-     
-    psElemType tileType;                // Type corresponding to "long"
-    if (sizeof(long) == sizeof(psS64)) {
-        tileType = PS_TYPE_S64;
-    } else if (sizeof(long) == sizeof(psS32)) {
-        tileType = PS_TYPE_S32;
-    } else {
-        psAbort("can't map (long) type to a psLib type");
-    }
-
-    sf->tiles = psVectorAlloc(3, tileType); // Tile sizes
-    if (tileType == PS_TYPE_S64) {
-        sf->tiles->data.S64[0] = 0;
-        sf->tiles->data.S64[1] = 1;
-        sf->tiles->data.S64[2] = 1;
-    } else {
-        sf->tiles->data.S32[0] = 0;
-        sf->tiles->data.S32[1] = 1;
-        sf->tiles->data.S32[2] = 1;
-    }
-
-    return sf;
-}
-
-
-bool
-streakFilesLock(streakFiles *sf)
-{
-    if (sf->inImage->inNebulous ) {
-        // TODO: make the nebulous call to lock the file
-    }
-    return true;
-}
-bool
-streakFilesUnlock(streakFiles *sf)
-{
-    if (sf->inImage->inNebulous ) {
-        // TODO: make the nebulous call to unlock the file
-    }
-    return true;
-}
-
-static bool
-moveExt(sFile *sfile, int extnum)
-{
-    bool status = psFitsMoveExtNum(sfile->fits, extnum, false);
-    if (!status) {
-        psError(PS_ERR_IO, false, 
-            "failed to move to extension %d for %s", extnum, sfile->resolved_name);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    return true;
-}
-
-static bool
-streakFilesNextExtension(streakFiles *sf)
-{
-    // unless stage is raw or chip when we get here we're done
-    if (!CHIP_LEVEL_INPUT(sf->stage)) {
-        sf->view->readout = -1;
-        pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
-        sf->view->cell = -1;
-        pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
-        sf->view->chip = -1;
-        pmFPAfileIOChecks(sf->config, sf->view, PM_FPA_AFTER);
-        return false;
-    }
-
-    ++sf->extnum;
-    if (sf->nHDU == 1) {
-        // return true the first time through
-        return (sf->extnum == 1);
-    }
-    if (sf->extnum < sf->nHDU) {
-        moveExt(sf->inImage, sf->extnum);
-        if (sf->inMask) {
-            moveExt(sf->inMask, sf->extnum);
-        }
-        if (sf->inWeight) {
-            moveExt(sf->inWeight, sf->extnum);
-        }
-        return true;
-    } else {
-        return false;
-    }
-}
-
-psString checkdir(char *path)
-{
-    // insure that path ends in /
-    int len = strlen(path);
-    psString dir = NULL;
-    if (strrchr(path, '/') != (path + len - 1)) {
-        psStringAppend(&dir, "%s/", path);
-    } else {
-        dir = psStringCopy(path);
-    }
-    return dir;
-}
-
-ippStage
-parseStage(psString stageStr)
-{
-    if (!strcmp("raw", stageStr)) {
-        return IPP_STAGE_RAW;
-    } else if (!strcmp("chip", stageStr)) {
-        return IPP_STAGE_CHIP;
-    } else if (!strcmp("warp", stageStr)) {
-        return IPP_STAGE_WARP;
-    } else if (!strcmp("diff", stageStr)) {
-        return IPP_STAGE_DIFF;
-    } else {
-        psError(PS_ERR_UNKNOWN, true, "unknown stage string: %s\n", stageStr);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        // notreached
-        return IPP_STAGE_NONE;
-    }
-}
-        
-
-pmConfig *parseArguments(int argc, char **argv) {
+            
+            if (sfiles->inImage->image) {
+                for (int i = 0; i < psArrayLength (pixels); ++i) {
+                    PixelPos *pixelPos = psArrayGet (pixels, i);
+
+                    if (!checkNonWarpedPixels || warpedPixel(sfiles, pixelPos)) {
+
+                        excisePixel(sfiles, pixelPos->x, pixelPos->y, true, maskStreak);
+
+                    } else {
+                        // This pixel was not included in any warp and has thus already excised
+                        // by exciseNonWarpedPixels
+                    }
+                }
+            } else { 
+                // this component contains an image cube, excise it completely
+                exciseImageCube = true;
+            }
+            psArrayElementsFree (pixels);
+        }
+
+        // write out the destreaked temporary images and the recovery images
+        writeImages(sfiles, exciseImageCube);
+
+    } while (streakFilesNextExtension(sfiles));
+
+    // close all files
+    closeImages(sfiles);
+
+    // NOTE: from here on we can't just quit if something goes wrong.
+    // especially if we're working at the raw stage
+
+    if (!replicateOutputs(sfiles)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
+        psErrorStackPrint(stderr, "");
+        exit(PS_EXIT_UNKNOWN_ERROR);
+    }
+
+#ifdef NOTYET
+    if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
+        //     swap the instances for the input and output
+        //     Note this is a database operation. No file I/O is performed
+        if (!swapOutputsToInputs(sfiles)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
+
+            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
+            // it has done and give a detailed report of what happened
+
+            psErrorStackPrint(stderr, "");
+            exit(PS_EXIT_UNKNOWN_ERROR);
+        }
+
+        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
+            //      delete the temporary storage objects (which now points to the original image(s)
+            if (!deleteTemps(sfiles)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
+                // XXX: Now what? At this point the output files have been swapped, so we can't
+                // repeat the operation.
+
+                // Returning error status here is problematic. The inputs have been streak removed
+                // but they're still lying around
+                // Maybe just print an error message and
+                // let other system tools clean up
+                psErrorStackPrint(stderr, "");
+                exit(PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
+#endif  // REPLACE, REMOVE
+    // nebServerFree(ourNebServer);
+    psFree(config);
+    pmConceptsDone();
+    pmConfigDone();
+    psLibFinalize();
+
+    //  PAU
+    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streaksremove");
+
+    return 0;
+}
+
+static void usage()
+{
+    fprintf(stderr, "USAGE: streaksremove -stage raw|chip|warp|diff -image IMAGE.fits -tmproot directory -streaks streaks_file\n\n");
+    fprintf(stderr, "Optional arguments:\n");
+    fprintf(stderr, "\t-class_id class_id: class_id (required for raw and chip stages)\n");
+    fprintf(stderr, "\t-recovery: directory to save excised pixel images\n");
+    fprintf(stderr, "\t-mask MASK.fits mask file to de-streak\n");
+    fprintf(stderr, "\t-weight WEIGHT.fits: weight file to de-streak\n");
+    fprintf(stderr, "\t-replace: replace the input images with the output\n");
+    fprintf(stderr, "\t-remove: remove the original image after processing (requires -replace)\n");
+    fprintf(stderr, "\t-keepnonwarped: do not exise pixels that were not part of difference processing\n");
+    fprintf(stderr, "\t-transparent val: instead of setting exicsed pixel to NAN add val\n");
+
+    exit(2);
+
+
+}
+static pmConfig *parseArguments(int argc, char **argv)
+{
 
     pmConfig *config = pmConfigRead(&argc, argv, NULL);
     if (!config) {
-        streaksremoveExit("failed to parse arguments", PS_EXIT_CONFIG_ERROR);
-    }
-
-    // bool status;
+        streaksExit("failed to parse arguments", PS_EXIT_CONFIG_ERROR);
+    }
+
     int argnum;
     ippStage stage = IPP_STAGE_NONE;
@@ -453,5 +222,5 @@
     } else {
         psError(PS_ERR_UNKNOWN, true, "-stage is required\n");
-        return NULL;
+        usage();
     }
 
@@ -463,5 +232,5 @@
     } else if ((stage == IPP_STAGE_RAW) || (stage == IPP_STAGE_CHIP)) {
         psError(PS_ERR_UNKNOWN, true, "-class_id is required\n");
-        return NULL;
+        usage();
     }
 
@@ -473,5 +242,5 @@
     } else {
         psError(PS_ERR_UNKNOWN, true, "-streaks is required\n");
-        return NULL;
+        usage();
     }
 
@@ -492,26 +261,48 @@
             if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "INPUT", "-image", NULL)) { ;
                 psError(PS_ERR_UNKNOWN, false, "failed to process -image");
-                return NULL;
+                usage();
             }
         }
     } else {
         psError(PS_ERR_UNKNOWN, true, "-image is required\n");
-        return NULL;
+        usage();
+    }
+
+    bool gotReplace = false;
+    if ((argnum = psArgumentGet(argc, argv, "-replace"))) {
+        gotReplace = true;
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files",
+            true);
     }
     
-    
-    if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "SKYCELLS", "-skycell", "-skycelllist")) { ;
-        if (CHIP_LEVEL_INPUT(stage)) {
-            psError(PS_ERR_UNKNOWN, true, "-skycelllist is required for raw and chip stages\n");
-            return NULL;
-        }
-    }
+    if ((argnum = psArgumentGet(argc, argv, "-keepnonwarped"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "KEEP_NON_WARPED", 0,
+            "skip excising of non warped pixels", true);
+    }
+
+    if ((argnum = psArgumentGet(argc, argv, "-transparent"))) {
+        if (gotReplace) {
+            psError(PS_ERR_UNKNOWN, true, "cannot make transparent streaks if -replace");
+            usage();
+        }
+        psArgumentRemove(argnum, &argc, argv);
+        double transparentStreaks = atof(argv[argnum]);
+        psMetadataAddF64(config->arguments, PS_LIST_TAIL, "TRANSPARENT_STREAKS", 0,
+            "value to adjust excised pixels", transparentStreaks);
+        psArgumentRemove(argnum, &argc, argv);
+    }
+        
+    // if skycells are not provided then we have to execise all pixels  unless -keepnonwarped
+    pmConfigFileSetsMD(config->arguments, &argc, argv, "SKYCELLS", "-skycell", "-skycelllist");
 
     if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", NULL)) { ;
         if (CHIP_LEVEL_INPUT(stage)) {
             psError(PS_ERR_UNKNOWN, true, "-astrom is required for raw and chip stages\n");
-            return NULL;
-        }
-    }
+            usage();
+        }
+    }
+
     if ((argnum = psArgumentGet(argc, argv, "-mask"))) {
         psArgumentRemove(argnum, &argc, argv);
@@ -520,10 +311,11 @@
             psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n",
                 nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
-            return NULL;
-        }
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "MASK", 0,
+            usage();
+        }
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0,
                 "name of input mask image", argv[argnum]);
         psArgumentRemove(argnum, &argc, argv);
     }
+
     if ((argnum = psArgumentGet(argc, argv, "-weight"))) {
         psArgumentRemove(argnum, &argc, argv);
@@ -532,19 +324,20 @@
             psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n",
                 nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
-            return NULL;
-        }
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "WEIGHT", 0,
+            usage();
+        }
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0,
                 "name of input weight image", argv[argnum]);
         psArgumentRemove(argnum, &argc, argv);
     }
+
     if ((argnum = psArgumentGet(argc, argv, "-tmproot"))) {
         psArgumentRemove(argnum, &argc, argv);
         bool nebulousTmp = IN_NEBULOUS(argv[argnum]);
-        if (nebulousTmp != nebulousImage) {
-            psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path\n",
+        if (gotReplace && (nebulousTmp != nebulousImage)) {
+            psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path with -replace\n",
                 nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
-            return NULL;
-        }
-        psString dir = checkdir(argv[argnum]);
+            usage();
+        }
+        psString dir = pathToDirectory(argv[argnum]);
         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files",
             dir);
@@ -552,27 +345,22 @@
     } else {
         psError(PS_ERR_UNKNOWN, true, "-tmproot is required\n");
-        return NULL;
-    }
+        usage();
+    }
+
     if ((argnum = psArgumentGet(argc, argv, "-recovery"))) {
         psArgumentRemove(argnum, &argc, argv);
-        psString dir = checkdir(argv[argnum]);
+        psString dir = pathToDirectory(argv[argnum]);
         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files",
             dir);
         psArgumentRemove(argnum, &argc, argv);
-    } else if (stage == IPP_STAGE_RAW) {
-        psError(PS_ERR_UNKNOWN, true, "-recovery is required for -stage raw\n");
-        return NULL;
-    }
-    bool gotReplace = false;
-    if ((argnum = psArgumentGet(argc, argv, "-replace"))) {
-        gotReplace = true;
-        psArgumentRemove(argnum, &argc, argv);
-        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files",
-            true);
-    }
+    } else if ((stage == IPP_STAGE_RAW) && gotReplace) {
+        psError(PS_ERR_UNKNOWN, true, "-recovery is required for -stage raw with -replace\n");
+        usage();
+    }
+
     if ((argnum = psArgumentGet(argc, argv, "-remove"))) {
         if (!gotReplace) {
             psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n");
-            return NULL;
+            usage();
         }
         psArgumentRemove(argnum, &argc, argv);
@@ -588,297 +376,46 @@
         psError(PS_ERR_UNKNOWN, true, "unexpected arguments: %s", unexpectedArguments);
         psFree(unexpectedArguments);
-        return NULL;
+        usage();
     }
 
     return config;
-}
-
-static void
-copyPHU(streakFiles *sfiles)
-{
-    psAssert(sfiles->stage == IPP_STAGE_RAW, "copyPHU should only be used for raw stage");
-
-    psMetadata *imageHeader = psFitsReadHeader(NULL, sfiles->inImage->fits);
-    if (!imageHeader) {
-        psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
-            sfiles->inImage->resolved_name);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    
-    // TODO: add keyword indicating that streaks have been removed
-    if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
-        psError(PS_ERR_IO, false, "failed to write primary header to %s",
-            sfiles->outImage->resolved_name);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    // TODO: add keyword indicating that this is the recovery image
-    if (!psFitsWriteBlank(sfiles->recImage->fits, imageHeader, NULL)) {
-        psError(PS_ERR_IO, false, "failed to write primary header to %s",
-            sfiles->recImage->resolved_name);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    psFree(imageHeader);
-
-    sFile *inMask = sfiles->inMask;
-    if (inMask) {
-        psMetadata *maskHeader = psFitsReadHeader(NULL, inMask->fits);
-        if (!maskHeader) {
-            psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
-                sfiles->inMask->resolved_name);
-            streaksremoveExit("", 1);
-        }
-        // TODO: add keyword indicating that streaks have been removed
-        if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
-            psError(PS_ERR_IO, false, "failed to write primary header to %s",
-                sfiles->outMask->resolved_name);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        // TODO: add keyword indicating that this is the recovery image
-        if (!psFitsWriteBlank(sfiles->recMask->fits, maskHeader, NULL)) {
-            psError(PS_ERR_IO, false, "failed to write primary header to %s",
-                sfiles->recMask->resolved_name);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        psFree(maskHeader);
-    }
-    sFile *inWeight = sfiles->inWeight;
-    if (inWeight) {
-        psMetadata *weightHeader = psFitsReadHeader(NULL, inWeight->fits);
-        if (!weightHeader) {
-            psError(PS_ERR_IO, false, "failed to read primary header from %s\n",
-                sfiles->inWeight->resolved_name);
-            streaksremoveExit("", 1);
-        }
-        // TODO: add keyword indicating that streaks have been removed
-        if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
-            psError(PS_ERR_IO, false, "failed to write primary header to %s",
-                sfiles->outWeight->resolved_name);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        // TODO: add keyword indicating that this is a recovery image
-        if (!psFitsWriteBlank(sfiles->recWeight->fits, weightHeader, NULL)) {
-            psError(PS_ERR_IO, false, "failed to write primary header to %s",
-                sfiles->recWeight->resolved_name);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        psFree(weightHeader);
-    }
-}
-
-// Determine whether the current header for this file is an image or not
-static bool
-isImage(sFile *in)
-{
-    bool status;
-    psString exttype = psMetadataLookupStr(&status, in->header, "EXTTYPE");
-    if (exttype && !strcmp(exttype, "IMAGE")) {
-        return true;
-    }
-    // examine current header and determine if it is an image
-    psString xtension = psMetadataLookupStr(&status, in->header, "XTENSION");
-    if (xtension) {
-        if (!strcmp(xtension, "IMAGE")) {
-            return true;
-        } else if (!strcmp(xtension, "BINTABLE")) {
-            psString ztension = psMetadataLookupStr(&status, in->header, "ZTENSION");
-            if (ztension && !strcmp(ztension, "IMAGE")) {
-                return true;
-            }
-        }
-    } else if (in->nHDU == 1) {
-        // no extensions in the file, can just return true? For now require 
-        // at least one dimension
-        int naxis =  psMetadataLookupS32(&status, in->header, "NAXIS");
-        return naxis > 0;
-    }
-    return false;
-}
-
-static void
-readImageFrom_pmFile(streakFiles *sf)
-{
-    // XXX: This function assumes that it is only used for a single
-    // chip single cell FPA (i.e. a skycell)
-    pmFPAview *view = sf->view;
-    assert(view->chip == 0);
-    view->cell = 0;
-    sf->cell =  pmFPAviewThisCell(view, sf->inImage->pmfile->fpa);
-
-    if (!sf->cell) {
-        streaksremoveExit("no cells found in chip", PS_EXIT_PROG_ERROR);
-    }
-    if (sf->cell->readouts->n != 1) {
-        psError(PS_ERR_PROGRAMMING, true, "unexpected number of readouts found: %ld", sf->cell->readouts->n);
-        streaksremoveExit("", PS_EXIT_PROG_ERROR);
-    }
-    view->readout = 0;
-    pmReadout *readout = pmFPAviewThisReadout(view, sf->inImage->pmfile->fpa);
-    if (!readout) {
-        streaksremoveExit("readout 0 not found", PS_EXIT_PROG_ERROR);
-    }
-
-    sf->inImage->image = (psImage*) psMemIncrRefCounter(readout->image);
-}
-
-static void
-readImage(sFile *in, int extnum)
-{
-    psRegion region = {0, 0, 0, 0};
-
-    if (in->header) psFree(in->header);
-    if (in->image) {
-        psFree(in->image);
-        in->image = NULL;
-    }
-    if (in->imagecube) {
-        psFree(in->imagecube);
-        in->imagecube = NULL;
-    }
-
-    in->header = psFitsReadHeader(NULL, in->fits);
-    if (!in->header) {
-        psError(PS_ERR_IO, false, "failed to read header from %s extnum: %d", 
-            in->resolved_name, extnum);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-
-    if (!isImage(in)) {
-        return;
-    }
-
-    in->numZPlanes = psMetadataLookupS32(NULL, in->header, "NAXIS3");
-
-    if (in->numZPlanes == 0) {
-        in->image = psFitsReadImage(in->fits, region, 0);
-        if (!in->image) {
-            psError(PS_ERR_IO, false, "failed to read image from %s extnum: %d", 
-                in->resolved_name, extnum);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        in->numCols = in->image->numCols;
-        in->numRows = in->image->numRows;
-    }  else {
-        in->imagecube = psFitsReadImageCube(in->fits, region);
-        if (!in->imagecube) {
-            psError(PS_ERR_IO, false, "failed to read image cube from %s extnum: %d", 
-                in->resolved_name, extnum);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
-        }
-        psImage *image = (psImage *) (in->imagecube->data[0]);
-        in->numCols = image->numCols;
-        in->numRows = image->numRows;
-    }
-
-}
-
-static void
-setFitsOptions(sFile *sfile, psString extname, int bitpix, float bscale, float bzero)
-{
-    if (sfile->fits->options) {
-        psFree(sfile->fits->options);
-    }
-    sfile->fits->options = psFitsOptionsAlloc();
-    sfile->fits->options->scaling = PS_FITS_SCALE_MANUAL;
-    sfile->fits->options->bitpix = bitpix;
-    sfile->fits->options->bscale = bscale;
-    sfile->fits->options->bzero = bzero;
-}
-
-static void
-copyFitsOptions(sFile *out, sFile *rec, sFile *in)
-{
-    // Get current BITPIX, BSCALE, BZERO, EXTNAME
-    // Probably not necessary to look the numerical values up in this
-    // way, but guards against changes to psLib and cfitsio FITS
-    // handling.
-    psMetadataItem *bitpixItem = psMetadataLookup(in->header, "BITPIX");
-    psAssert(bitpixItem, "Every FITS image should have BITPIX");
-    int bitpix = psMetadataItemParseS32(bitpixItem);
-    psMetadataItem *bscaleItem = psMetadataLookup(in->header, "BSCALE");
-
-    float bscale;
-    if (!bscaleItem) {
-        psWarning("BSCALE isn't set; defaulting to unity");
-        bscale = 1.0;
-    } else {
-        bscale = psMetadataItemParseF32(bscaleItem);
-    }
-    psMetadataItem *bzeroItem = psMetadataLookup(in->header, "BZERO");
-    float bzero;
-    if (!bzeroItem) {
-        psWarning("BZERO isn't set; defaulting to zero");
-        bzero = 0.0;
-    } else {
-        bzero = psMetadataItemParseF32(bzeroItem);
-    }
-    bool mdok;
-
-    psString extname = psMetadataLookupStr(&mdok, in->header, "EXTNAME");
-    setFitsOptions(out, extname, bitpix, bscale, bzero);
-    setFitsOptions(rec, extname, bitpix, bscale, bzero);
-}
-
-
-static void
-copyTable(sFile *out, sFile *in, int extnum)
-{
-    bool mdok;
-    psString xtension = psMetadataLookupStr(&mdok, in->header, "XTENSION");
-    psString extname = psMetadataLookupStr(&mdok, in->header, "EXTNAME");
-
-    if (!xtension) {
-        psWarning("extnum %d has no image and xtension is not defined in %s",
-            extnum, in->resolved_name);
-        // XXX abort?
-        return;
-    } 
-    if (!extname) {
-        psWarning("extnum %d has no image and extname not defined in %s",
-            extnum, in->resolved_name);
-        // XXX abort?
-        return;
-    }
-    psArray *table = psFitsReadTable(in->fits);
-    if (!table) {
-        psError(PS_ERR_UNKNOWN, false, "failed to read table in extension %d from in->resolved name", extnum);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-
-    if (!psFitsWriteTable(out->fits, out->header, table, extname)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to copy table in extension %d", extnum);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-}
-
-static void
-createRecoveryImage(sFile *rec, sFile *in, int extnum)
-{
-    rec->image = psImageRecycle(rec->image, in->image->numCols, in->image->numRows, in->image->type.type);
-    if (!rec->image) {
-        psError(PS_ERR_UNKNOWN, false, "failed to allocate recoveyr image for extnsion %d", extnum);
-        streaksremoveExit("", PS_EXIT_UNKNOWN_ERROR);
-    }
-    psImageInit(rec->image, 0);
 }
 
 // set the image for the output files to the contents of the corresponding input file.
 static bool
-readAndCopyToOutput(streakFiles *sf)
-{
+readAndCopyToOutput(streakFiles *sf, bool exciseAll)
+{
+    bool    updateAstrometry = false;
     if (sf->inImage->pmfile) {
         // image data from pmFPAfile (diff or warp file)
         readImageFrom_pmFile(sf);
-        sf->astrom = streakSetAstrometry(sf->astrom, sf->inAstrom->fpa, sf->chip, true, sf->cell->concepts,
-            sf->inImage->numCols, sf->inImage->numRows);
+        sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, sf->inAstrom->fpa,
+                sf->chip, true, sf->cell->concepts, sf->inImage->numCols, sf->inImage->numRows);
     } else {
         // image data directly from psFits
-        readImage(sf->inImage, sf->extnum);
+        readImage(sf->inImage, sf->extnum, sf->stage);
         if (SFILE_IS_IMAGE(sf->inImage)) {
-            sf->astrom = streakSetAstrometry(sf->astrom, sf->inAstrom->fpa, sf->chip, false,
+            sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, sf->inAstrom->fpa, sf->chip, false,
                 sf->inImage->header, sf->inImage->numCols, sf->inImage->numRows);
         }
     }
+    if (!sf->astrom) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set up astrometry for extension %d", sf->extnum);
+        streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+    }
+    if (sf->stage == IPP_STAGE_CHIP) {
+        // For the chip level files, copy the WCS from the astrometry file to the header
+        if (!sf->bilevelAstrometry) {
+            updateAstrometry = true;
+            if (!pmAstromWriteWCS(sf->inImage->header, sf->inAstrom->fpa, sf->chip, 0.001)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to update astrometry for extension %d", sf->extnum);
+                streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
     sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
-    sf->recImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
+    if (sf->recImage) {
+        sf->recImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
+    }
 
     if (!SFILE_IS_IMAGE(sf->inImage)) {
@@ -890,11 +427,13 @@
 
     if (sf->inImage->image) {
-        sf->outImage->image = (psImage*) psMemIncrRefCounter(sf->inImage->image);
-        createRecoveryImage(sf->recImage, sf->inImage, sf->extnum);
+        setupImageRefs(sf->outImage, sf->recImage, sf->inImage, sf->extnum, exciseAll);
     }  else {
-        // we don't do a reference if we have an imagecube
+        // Image cubes are handeled specially
     }
 
     // set up the compression parameters
+#ifdef STREAKS_COMPRESS_OUTPUT
+    // compression of the image pixels is disabled for now. Some consortium members
+    // have problems reading them
     copyFitsOptions(sf->outImage, sf->recImage, sf->inImage);
 
@@ -903,81 +442,100 @@
     // perhaps we should just use the definition of COMP_IMG in the configuration 
     psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
-    psFitsSetCompression(sf->recImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
+    if (sf->recImage) {
+        psFitsSetCompression(sf->recImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
+    }
+#endif
 
     if (sf->inMask) {
-        readImage(sf->inMask, sf->extnum);
+        readImage(sf->inMask, sf->extnum, sf->stage);
         sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
-        sf->outMask->image = (psImage*) psMemIncrRefCounter(sf->inMask->image);
-        sf->recMask->image = (psImage*) psMemIncrRefCounter(sf->inMask->image);
-
-        createRecoveryImage(sf->recMask, sf->inMask, sf->extnum);
+        if (sf->recMask) {
+            sf->recMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+        }
+        if (updateAstrometry) {
+            pmAstromWriteWCS(sf->outMask->header, sf->inAstrom->fpa, sf->chip, 0.001);
+        }
+
+        setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, exciseAll);
+
+#ifdef STREAKS_COMPRESS_OUTPUT
+        // XXX: see note above
         copyFitsOptions(sf->outMask, sf->recMask, sf->inMask);
-
-        // XXX: see note above
         psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
-        psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+        if (sf->recMask) {
+            psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+        }
+#endif
     }
 
     if (sf->inWeight) {
-        readImage(sf->inWeight, sf->extnum);
+        readImage(sf->inWeight, sf->extnum, sf->stage);
         sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
-        sf->outWeight->image = (psImage*) psMemIncrRefCounter(sf->inWeight->image);
-
-        sf->recWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
-        createRecoveryImage(sf->recWeight, sf->inWeight, sf->extnum);
-
+        if (sf->recWeight) {
+            sf->recWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
+        }
+        if (updateAstrometry) {
+            pmAstromWriteWCS(sf->inWeight->header, sf->inAstrom->fpa, sf->chip, 0.001);
+        }
+        setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll);
+
+#ifdef STREAKS_COMPRESS_OUTPUT
         copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight);
         // XXX: see note above
         psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
-        psFitsSetCompression(sf->recWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
-    }
-
+        if (sf->recWeight) {
+            psFitsSetCompression(sf->recWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
+        }
+#endif
+    }
     // and for raw images, create sub images that represent the actual image
     // area (no overscan)
 
-
     return true;
 }
 
+
+
 static void
-writeImage(sFile *sfile, psString extname, int extnum)
-{
-    if (!psFitsWriteImage(sfile->fits, sfile->header, sfile->image, 0, extname)) {
-        psError(PS_ERR_IO, false, "failed to write image to %s extnum: %d", 
-            sfile->resolved_name, extnum);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    psFree(sfile->image);
-    sfile->image = NULL;
-    psFree(sfile->header);
-    sfile->header = NULL;
-}
-static void
-writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum)
-{
-    if (!psFitsWriteImageCube(sfile->fits, sfile->header, imagecube, extname)) {
-        psError(PS_ERR_IO, false, "failed to write image to %s extnum: %d", 
-            sfile->resolved_name, extnum);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    psFree(sfile->header);
-    sfile->header = NULL;
-}
-
-static void
-writeImages(streakFiles *sf)
+writeImages(streakFiles *sf, bool exciseImageCube)
 {
     psString extname = NULL;
     if (sf->nHDU > 1) {
-        extname = psMetadataLookupStr(NULL, sf->inImage->header, "EXTNAME");
+        bool mdok;
+        extname = psMetadataLookupStr(&mdok, sf->inImage->header, "EXTNAME");
     }
     if (sf->inImage->numZPlanes == 0)  {
+        // note exciseing complete images is handled in readAndCopyToOutput
         writeImage(sf->outImage, extname, sf->extnum);
         writeImage(sf->recImage, extname, sf->extnum);
     } else {
-        // XXX: TODO: if a streak touched this cell then
-        // write the image cube to the recovery image if no streaks write it
-        // to the output image
-        writeImageCube(sf->outImage, sf->inImage->imagecube, extname, sf->extnum);
+        // we have an image cube
+        double initValue;
+        if (exciseImageCube) {
+            // copy the entire input image to the recovery image
+            writeImageCube(sf->recImage, sf->inImage->imagecube, extname, sf->extnum);
+            initValue = NAN;
+        } else {
+            // otherwise write it to the output 
+            writeImageCube(sf->outImage, sf->inImage->imagecube, extname, sf->extnum);
+            initValue = 0;
+        }
+
+        // borrow one of the images from the imagecube and set it to init value
+        psImage *image = psArrayGet (sf->inImage->imagecube, 0);
+        psMemIncrRefCounter(image);
+        psImageInit(image, initValue);
+        if (exciseImageCube) {
+            sf->outImage->image = image;
+            writeImage(sf->outImage, extname, sf->extnum);
+        } else {
+            // write zero valued image to reccovery
+            if (sf->recImage) {
+                sf->recImage->image = image;
+                writeImage(sf->recImage, extname, sf->extnum);
+            }
+        }
+        // Assumption: there are no mask and weight images for video cells
+        return;
     }
     if (sf->outMask) {
@@ -989,61 +547,4 @@
         writeImage(sf->recWeight, extname, sf->extnum);
     }
-}
-
-static void
-closeImage(sFile *sfile)
-{
-    if (!sfile->fits) {
-        return;
-    }
-    if (!psFitsClose(sfile->fits)) {
-        psError(PS_ERR_IO, false, "failed to close image to %s", 
-            sfile->resolved_name);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
-    }
-    psFree(sfile->header);
-    psFree(sfile->image);
-    psFree(sfile->imagecube);
-    psFree(sfile->name);
-    psFree(sfile->resolved_name);
-    psFree(sfile);
-}
-
-static void
-closeImages(streakFiles *sf)
-{
-    closeImage(sf->inImage);
-    closeImage(sf->outImage);
-    closeImage(sf->recImage);
-    if (sf->inMask) {
-        closeImage(sf->inMask);
-        closeImage(sf->outMask);
-        closeImage(sf->recMask);
-    }
-    if (sf->inWeight) {
-        closeImage(sf->inWeight);
-        closeImage(sf->outWeight);
-        closeImage(sf->recWeight);
-    }
-}
-
-static bool
-replicate(sFile *sfile, void *xattr)
-{
-    if (!sfile->inNebulous) {
-        return true;
-    }
-    nebServer *server = getNebServer(NULL);
-
-    // for now just set "user.copies" to 2
-    if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
-        return false;
-    }
-    if (!nebReplicate(server, sfile->name, NULL, NULL)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
-        return false;
-    }
-    return true;
 }
 
@@ -1093,261 +594,195 @@
 }
 
+
+static void
+excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue)
+{
+    // we clip so that the streak calculation code doesn't have to
+    if ((x < 0) || (x >= sfiles->inImage->numCols) || (y < 0) || (y >= sfiles->inImage->numRows)) {
+        return;
+    }
+
+    double imageValue  = psImageGet (sfiles->inImage->image,  x, y);
+    if (sfiles->recImage && !isnan(imageValue) ) {
+        psImageSet (sfiles->recImage->image,  x, y, imageValue);
+    }
+
+    if (sfiles->transparentStreaks == 0) {
+        psImageSet (sfiles->outImage->image,  x, y, NAN);
+    } else {
+        if (streak) {
+            // as a visualization aid don't mask the pixel, just change the intensity
+            psImageSet (sfiles->outImage->image,  x, y, imageValue + sfiles->transparentStreaks);
+        } else {
+            psImageSet (sfiles->outImage->image,  x, y, NAN);
+        }
+    }
+
+    if (sfiles->inWeight) {
+        if (sfiles->recWeight) {
+            double weightValue = psImageGet (sfiles->inWeight->image, x, y);
+            psImageSet (sfiles->recWeight->image, x, y, weightValue);
+        }
+        psImageSet (sfiles->outWeight->image, x, y, NAN);
+    }
+    if (sfiles->inMask) {
+        if (sfiles->recMask) {
+            double maskValue   = psImageGet (sfiles->inMask->image,   x, y);
+            psImageSet (sfiles->recMask->image,   x, y, maskValue);
+        }
+        psImageSet (sfiles->outMask->image,   x, y, newMaskValue);
+    }
+}
+
+static void
+exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue)
+{
+    int cell_x0 = sfiles->astrom->cell_x0;
+    int cell_y0 = sfiles->astrom->cell_y0;
+    int xParity = sfiles->astrom->xParity;
+    int yParity = sfiles->astrom->yParity;
+    int numCols = sfiles->inImage->numCols;
+    int numRows = sfiles->inImage->numRows;
+
+//    printf("%2d x0: %4d y0: %4d xpar: %d ypar: %d\n", sfiles->extnum, cell_x0, cell_y0, xParity, yParity);
+
+    for (int yCell = 0; yCell < numRows; yCell++) {
+        int yChip;
+        if (yParity == 1) {
+            yChip = cell_y0 + yCell;
+        } else {
+            yChip = cell_y0 - yCell;
+        }
+
+        psU8 *pixels = sfiles->warpedPixels->data.U8[yChip];
+
+        if (xParity == 1) {
+            pixels += cell_x0;
+            for (int xCell = 0; xCell < numCols; xCell++, pixels++) {
+                if (! *pixels ) {
+                    excisePixel(sfiles, xCell, yCell, false, newMaskValue);
+                }
+            }
+        } else {
+            // negative parity
+            // point to the right most pixel in chip coordinates (lowest in cell coords)
+            pixels += cell_x0 - numCols;
+            for (int xCell = numCols - 1; xCell >= 0 ; xCell--, pixels++) {
+                if (!*pixels) {
+                    excisePixel(sfiles, xCell, yCell, false, newMaskValue);
+                }
+            }
+        }
+    }
+}
+
 static bool
-swapOutputToInput(sFile *in, sFile *out)
-{
-
-    if (in->inNebulous) {
-        nebServer *server = getNebServer(NULL);
-
-        if (!nebSwap(server, in->name, out->name)) {
-            psError(PM_ERR_SYS, true, "failed to swap files for: %s.",
-                in->name, nebErr(server));
-            return false;
-        }
-    } else {
-        psString command = NULL;
-
-        // regular files. use mv creating a backup
-        // NOTE: -b is a gnu option and may not always be available
-        psStringAppend(&command, "mv -b --suffix=.bak %s %s", out->resolved_name, in->resolved_name);
-        int status = system(command);
-        if (status != 0) {
-            psError(PM_ERR_SYS, true, "%s failed with status %d.",
-                command, status);
-            psFree(command);
-            return false;
-        }
-        psFree(command);
-    }
-
-    return true;
-}
+warpedPixel(streakFiles *sfiles, PixelPos *cellCoord)
+{
+    PixelPos chipCoord;
+
+    if (!CHIP_LEVEL_INPUT(sfiles->stage)) {
+        // if we're here on a skycell image by definition this pixel was warped
+        return true;
+    }
+
+    // we clip so that the streak calculation code doesn't have to
+    if ((cellCoord->x < 0) || (cellCoord->x >= sfiles->inImage->numCols) ||
+        (cellCoord->y < 0) || (cellCoord->y >= sfiles->inImage->numRows)) {
+        return false;
+    }
+
+    cellToChip(&chipCoord, sfiles->astrom, cellCoord);
+
+#ifdef notdef
+    // Shouldn't need this but for raw images I'm getting 
+    // x = warpedPixels->numCols for cell xy70 when cell.x = 0
+    // Perhaps cell_x0 should be IMNPIX-1 since IMNPIX is in the 1 based system of the fits
+    // headers
+    if (chipCoord.x < 0 || chipCoord.x >= sfiles->warpedPixels->numCols) {
+        return false;
+    }
+    if (chipCoord.y < 0 || chipCoord.y >= sfiles->warpedPixels->numRows) {
+        return false;
+    }
+#endif
+
+    return psImageGet(sfiles->warpedPixels, chipCoord.x, chipCoord.y) ? true : false;
+}
+
 static bool
-swapOutputsToInputs(streakFiles *sfiles)
-{
-    if (sfiles->inMask) {
-        if (!swapOutputToInput(sfiles->inMask, sfiles->outMask)) {
-            psError(PM_ERR_SYS, false, "failed to swap instances for Mask.");
-            return false;
-        }
-    }
-
-    if (sfiles->inWeight) {
-        if (!swapOutputToInput(sfiles->inWeight, sfiles->outWeight)) {
-            psError(PM_ERR_SYS, false, "failed to swap instances for Weight.");
-            return false;
-        }
-    }
-
-    if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
-        psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
-        return false;
-    }
-
-    return true;
-}
-
-static bool
-deleteFile(sFile *sfile)
-{
-
-    if (sfile->inNebulous) {
-        nebServer *server = getNebServer(NULL);
-        if (!nebDelete(server, sfile->name)) {
-            psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfile->name,
-                nebErr(server));
-            return false;
-        }
-    } else {
-        if (unlink(sfile->resolved_name)) {
-            psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfile->resolved_name,
-                strerror(errno));
-            return false;
-        }
-    }
-    return true;
-}
-
-static bool
-deleteTemps(streakFiles *sfiles)
-{
-    if (sfiles->outMask) {
-        if (!deleteFile(sfiles->outMask)) {
-            psError(PM_ERR_SYS, false, "failed to delete Mask.");
-            return false;
-        }
-    }
-
-    if (sfiles->outWeight) {
-        if (!deleteFile(sfiles->outWeight)) {
-            psError(PM_ERR_SYS, false, "failed to delete Weight.");
-            return false;
-        }
-    }
-
-    if (!deleteFile(sfiles->outImage)) {
-        psError(PM_ERR_SYS, false, "failed to delete Image.");
-        return false;
-    }
-
-    return true;
-}
-
-int
-main(int argc, char *argv[])
-{
-    long i;
-    double imageValue, weightValue, maskValue;
+streakEndOnComponent(streak *streak, double r_min, double r_max, double d_min, double d_max)
+{
+    if ((streak->ra1  >= r_min) && (streak->ra1  <= r_max) &&
+        (streak->dec1 >= d_min) && (streak->dec1 <= d_max)) {
+        return true;
+    }
+    if ((streak->ra2  >= r_min) && (streak->ra2  <= r_max) &&
+        (streak->dec2 >= d_min) && (streak->dec2 <= d_max)) {
+        return true;
+    }
+    return false;
+}
+
+
+static Streaks *
+restrictStreaksToComponent(streakFiles *sfiles, Streaks *streaks)
+{
+    Streaks *out = psAlloc(sizeof(Streaks));
+
+    strkPt ll, ur;
+    cellToSky(&ll, sfiles->astrom, 0, 0);
+    cellToSky(&ur, sfiles->astrom, sfiles->inImage->numCols-1, sfiles->inImage->numRows-1);
+
+    double r_min, r_max, d_min, d_max;
+    if (ll.x < ur.x) {
+        r_min = ll.x; r_max = ur.x;
+    } else {
+        r_min = ur.x; r_max = ll.x;
+    }
+    if (ll.y < ur.y) {
+        d_min = ll.y; d_max = ur.y;
+    } else {
+        d_min = ur.y; d_max = ll.y;
+    }
+
+    out->list = psAlloc(streaks->size * sizeof(streak));
+    int j=0;
+    for (int i=0 ; i<streaks->size; i++) {
+        if (streakEndOnComponent(&streaks->list[i], r_min, r_max, d_min, d_max)) {
+            out->list[j++] = streaks->list[i];
+        }
+            
+    }
+    out->size = j;
+    return out;
+}
+
+static StreakPixels * 
+getStreakPixels(streakFiles *sfiles, Streaks *streaks)
+{
     StreakPixels *pixels;
-    PixelPos *pixelPos;
-
-    psLibInit(NULL);
-
-    pmConfig *config = parseArguments(argc, argv);
-    if (!config) {
-        psError(PS_ERR_UNKNOWN, false, "failed to parse arguments\n");
-        return PS_EXIT_CONFIG_ERROR;
-    }
-
-    psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS");
-    Streaks *streaks = readStreaksFile(streaksFileName);
-
-    if (!streaks) {
-        psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName);
-        exit (PS_EXIT_PROG_ERROR);
-    }
-
-    streakFiles *sfiles = openFiles(config);
-
-    if (sfiles->stage == IPP_STAGE_RAW) {
-        // copy PHU to output files
-        copyPHU(sfiles);
-        // advance to the first image extension
-        if (!streakFilesNextExtension(sfiles)) {
-            psErrorStackPrint(stderr, "failed to advance to first extension of input images");
-            exit (PS_EXIT_PROG_ERROR);
-        }
-    }
-
-    // Iterate through components of image
-    do {
-
-        // read the images and copy the pixels image from the inputs to the outputs
-        // This also sets up the astrometry and initializes the recovery images
-        if (!readAndCopyToOutput(sfiles)) {
-            // this extension is not an image skip (video descriptor? 
-            // XXX is there anything else that could be in an input file that we need to handle?
-            continue;
-        }
-
-        // Identify pixels to mask because of streaks
-
+#ifdef notyet
+    Streaks *componentStreaks = NULL;
+    Streaks *streaksToProcess;
+
+    bool only_component_streaks = false; // XXX flesh this out and make it an option
+    if (only_component_streaks) {
+        componentStreaks = restrictStreaksToComponent(sfiles, streaks);
+        streaksToProcess = componentStreaks;
+    } else { 
+        streaksToProcess = streaks;
+    }
+
+    pixels = streak_on_component (streaksToProcess, sfiles->astrom,
+                                  sfiles->inImage->numCols, sfiles->inImage->numRows);
+    if (componentStreaks) {
+        psFree(componentStreaks);
+    }
+#else
         pixels = streak_on_component (streaks, sfiles->astrom,
                                       sfiles->inImage->numCols, sfiles->inImage->numRows);
-
-#ifdef notyet
-        // XXX: How is this formatted?
-        // PFS: Aren't these chips that are never warped?  They should be cleared
-        //      entirely or copy them to archive location.  I don't know how
-        //      to find these chips.  The fact that a warp is not created for
-        //      a chip must be recorded somewhere.  I don't like this approach
-        //      of including everything (ALL) and then removing them if they
-        //      are warped.  I don't know if it is practical.
-        // From ICD:
-        // In the raw and detrended images, the pixels which were not
-        // included in any of the streak-processed warps must also be masked.
-        // Note that the warp and difference skycells are only generated if more
-        // than a small fraction of the pixels are lit by the input image.
-
-        // Identify pixels to mask because not in a warp
-        warp_pixels = pixel_list_initialise(ALL); // List of pixels to be excised because not in a warp
-        foreach warp (warps) {
-            // Calculate warp region on image
-            image_warp = warp_on_component(warp, astrom, numCols, numRows);
-            foreach pixel specified by image_warp {
-                remove_pixel(warp_pixels, pixel);
-            }
-        }
 #endif
-
-        // XXX: This loop following will SEGV for raw video Cells becuase the image is null
-        for (i = 0; i < psArrayLength (pixels); ++i) {
-            pixelPos = psArrayGet (pixels, i);
-            imageValue  = psImageGet (sfiles->inImage->image,  pixelPos->x, pixelPos->y);
-            psImageSet (sfiles->recImage->image,  pixelPos->x, pixelPos->y, imageValue);
-            psImageSet (sfiles->outImage->image,  pixelPos->x, pixelPos->y, NAN);
-            if (sfiles->inWeight) {
-                weightValue = psImageGet (sfiles->inWeight->image, pixelPos->x, pixelPos->y);
-                psImageSet (sfiles->recWeight->image, pixelPos->x, pixelPos->y, weightValue);
-                psImageSet (sfiles->outWeight->image, pixelPos->x, pixelPos->y, NAN);
-            }
-            if (sfiles->inMask) {
-                maskValue   = psImageGet (sfiles->inMask->image,   pixelPos->x, pixelPos->y);
-                psImageSet (sfiles->recMask->image,   pixelPos->x, pixelPos->y, maskValue);
-                // TODO:
-                // Need to get mask weight for PM_MASK_DETECTOR.  Is this stored
-                // in a config somewhere?  Not sure how to properly set the maskValue.
-                psImageSet (sfiles->outMask->image,   pixelPos->x, pixelPos->y, maskValue);
-            }
-        }
-        psArrayElementsFree (pixels);
-
-        // write out the destreaked temporary images and the recovery images
-        writeImages(sfiles);
-    } while (streakFilesNextExtension(sfiles));
-
-    // close all files
-    closeImages(sfiles);
-
-    // NOTE: from here on we can't just quit if something goes wrong.
-    // especially if we're working at the raw stage
-
-    if (!replicateOutputs(sfiles)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
-        psErrorStackPrint(stderr, "");
-        exit(PS_EXIT_UNKNOWN_ERROR);
-    }
-
-    bool status;
-    if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
-        //     swap the instances for the input and output
-        //     Note this is a database operation. No file I/O is performed
-        if (!swapOutputsToInputs(sfiles)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
-
-            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
-            // it has done and give a detailed report of what happened
-
-            psErrorStackPrint(stderr, "");
-            exit(PS_EXIT_UNKNOWN_ERROR);
-        }
-
-        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
-            //      delete the temporary storage objects (which now points to the original image(s)
-            if (!deleteTemps(sfiles)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
-                // XXX: Now what? At this point the output files have been swapped, so we can't
-                // repeat the operation.
-
-                // Returning error status here is problematic. The inputs have been streak removed
-                // but they're still lying around
-                // Maybe just print an error message and
-                // let other system tools clean up
-                psErrorStackPrint(stderr, "");
-                exit(PS_EXIT_UNKNOWN_ERROR);
-            }
-        }
-    }
-    psFree(ourNebServer);
-    psFree(config);
-    pmConceptsDone();
-    pmConfigDone();
-    psLibFinalize();
-#ifdef notyet
-#endif
-
-    //  PAU
-    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streaksremove");
-
-    return 0;
-}
+    return pixels;
+}
+
Index: branches/bills_081204/magic/remove/src/streaksremove.h
===================================================================
--- branches/bills_081204/magic/remove/src/streaksremove.h	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streaksremove.h	(revision 20890)
@@ -0,0 +1,87 @@
+#ifndef STREAKS_H
+#define STREAKS_H
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "string.h"
+#include "pslib.h"
+#include "psmodules.h"
+#include "nebclient.h"
+
+#include "streaksextern.h"
+#include "streaksastrom.h"
+
+typedef struct {
+    psString    resolved_name;
+    psString    name;
+    bool        inNebulous;
+    psFits      *fits;
+    int         nHDU;
+    psString    extname;
+    psMetadata  *header;
+    int         numZPlanes;
+    psImage     *image;
+    psArray     *imagecube;
+    pmFPAfile   *pmfile;
+    int         numCols;
+    int         numRows;
+} sFile;
+
+
+typedef enum {
+    IPP_STAGE_NONE = 0,
+    IPP_STAGE_RAW,
+    IPP_STAGE_CHIP,
+    IPP_STAGE_WARP,
+    IPP_STAGE_DIFF
+} ippStage;
+
+
+typedef struct {
+    pmConfig *config;
+    ippStage stage;
+    int     extnum;
+    int     nHDU;   // number of HDUs in the inputs (all must be equal)
+    sFile *inImage;
+    sFile *inMask;
+    sFile *inWeight;
+    sFile *outImage;
+    sFile *outMask;
+    sFile *outWeight;
+    sFile *recImage;
+    sFile *recMask;
+    sFile *recWeight;
+    psString class_id;
+    pmFPAfile  *inAstrom;
+    strkAstrom *astrom;
+    bool  bilevelAstrometry;
+    pmFPAview *view;
+    pmChip  *chip;  // current chip
+    pmCell  *cell;  // current cell
+    psImage *warpedPixels;
+    psVector    *tiles;
+    double  transparentStreaks;
+} streakFiles;
+
+#include "streaksio.h"
+
+extern void setupAstrometry(streakFiles *);
+extern strkAstrom *streakSetAstrometry(strkAstrom *, ippStage stage, pmFPA *, pmChip *, bool fromCell, psMetadata *md,
+    int numCols, int numRows);
+// can't declare this in streaksastrom due to header file ordering
+extern void cellToChip(PixelPos *chip, strkAstrom *astrom, PixelPos *cell);
+
+extern bool computeWarpedPixels(streakFiles *sf);
+extern void streaksExit(psString, int);
+extern ippStage parseStage(psString);
+extern psString pathToDirectory(char *path);
+
+#define CHIP_LEVEL_INPUT(_stage) ((_stage == IPP_STAGE_RAW) || (_stage == IPP_STAGE_CHIP))
+#define USE_SUPPLIED_ASTROM(_stage) CHIP_LEVEL_INPUT(_stage)
+
+#define SFILE_IS_IMAGE(_sfile) (_sfile->image || _sfile->imagecube)
+#define IN_NEBULOUS(_filename) (!strncasecmp(_filename, "neb://", strlen("neb://")))
+
+#endif // STREAKS_H
Index: branches/bills_081204/magic/remove/src/streaksreplace.c
===================================================================
--- branches/bills_081204/magic/remove/src/streaksreplace.c	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streaksreplace.c	(revision 20890)
@@ -0,0 +1,472 @@
+#include "streaksremove.h"
+
+static pmConfig *parseArguments(int, char **);
+static bool readAndCopyToOutput(streakFiles *sf, bool restoreImageCube);
+static void writeImages(streakFiles *sf, bool exciseImageCube);
+static bool replicateOutputs(streakFiles *sfiles);
+static bool readAndCopyToOutput(streakFiles *sf, bool restoreImageCube);
+
+int main(int argc, char *argv[])
+{
+    long i;
+    bool status;
+    StreakPixels *pixels;
+    PixelPos *pixelPos;
+
+    psLibInit(NULL);
+
+    pmConfig *config = parseArguments(argc, argv);
+    if (!config) {
+        psError(PS_ERR_UNKNOWN, false, "failed to parse arguments\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+    // TODO: perhaps we should save this in the image header and get it from there?
+    double maskStreak = (double) psMetadataLookupU8(&status, masks, "STREAK");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    streakFiles *sfiles = openFiles(config, false);
+
+    if (sfiles->stage == IPP_STAGE_RAW) {
+        // copy PHU to output files
+        copyPHU(sfiles, false);
+
+        // advance to the first image extension
+        if (!streakFilesNextExtension(sfiles)) {
+            psErrorStackPrint(stderr, "failed to advance to first extension of input images");
+            exit (PS_EXIT_PROG_ERROR);
+        }
+    }
+
+    // Iterate through components of input files
+    do {
+        bool restoreImageCube = false;
+
+        // read the images and copy the data from the inputs to the outputs
+        // This also sets up the astrometry and initializes the recovery image(s)
+
+        if (!readAndCopyToOutput(sfiles, restoreImageCube)) {
+            // false return value indicates that this extension is not an image and
+            // the contents has already been copied to the output file.
+            // we don't need to process this extesion for streaks.
+            continue;
+        }
+
+        // If pixel in recovery image is not NAN, then it contains an excised pixel
+        // that has been stored in the recovery image
+        for (int y=0; y < sfiles->inImage->numRows; y++) {
+            for (int x=0; x < sfiles->inImage->numCols; x++) {
+                double recValue = psImageGet(sfiles->recImage->image, x, y);
+                if (!isnan(recValue)) {
+                    psImageSet(sfiles->inImage->image, x, y, recValue);
+                }
+                if (sfiles->inMask) {
+                    double recMaskValue = psImageGet(sfiles->recMask->image, x, y);
+                    if (recValue != maskStreak) {
+                        psImageSet(sfiles->inMask->image, x, y, recMaskValue);
+                    }
+                }
+                if (sfiles->inWeight) {
+                    double recWeightValue = psImageGet(sfiles->recWeight->image, x, y);
+                    if (recWeightValue != NAN) {
+                        psImageSet(sfiles->inWeight->image, x, y, recWeightValue);
+                    }
+                }
+            }
+        }
+
+        // write out the re-streaked images
+        writeImages(sfiles, restoreImageCube);
+
+    } while (streakFilesNextExtension(sfiles));
+
+    // close all files
+    closeImages(sfiles);
+
+    // NOTE: from here on we can't just quit if something goes wrong.
+    // especially if we're working at the raw stage
+
+    if (!replicateOutputs(sfiles)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
+        psErrorStackPrint(stderr, "");
+        exit(PS_EXIT_UNKNOWN_ERROR);
+    }
+
+#ifdef NOTYET
+    if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
+        //     swap the instances for the input and output
+        //     Note this is a database operation. No file I/O is performed
+        if (!swapOutputsToInputs(sfiles)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
+
+            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
+            // it has done and give a detailed report of what happened
+
+            psErrorStackPrint(stderr, "");
+            exit(PS_EXIT_UNKNOWN_ERROR);
+        }
+
+        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
+            //      delete the temporary storage objects (which now points to the original image(s)
+            if (!deleteTemps(sfiles)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
+                // XXX: Now what? At this point the output files have been swapped, so we can't
+                // repeat the operation.
+
+                // Returning error status here is problematic. The inputs have been streak removed
+                // but they're still lying around
+                // Maybe just print an error message and
+                // let other system tools clean up
+                psErrorStackPrint(stderr, "");
+                exit(PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
+#endif  // REPLACE, REMOVE
+//    nebServerFree(ourNebServer);
+    psFree(config);
+    pmConceptsDone();
+    pmConfigDone();
+    psLibFinalize();
+
+    //  PAU
+    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streaksreplace");
+
+    return 0;
+}
+static void usage(void)
+{
+    fprintf(stderr, "USAGE: streaksreplace -stage raw|chip|warp|diff -image IMAGE.fits -tmproot directory\n\n");
+    fprintf(stderr, "Optional arguments:\n");
+    fprintf(stderr, "\t-mask MASK.fits mask file to re-streak\n");
+    fprintf(stderr, "\t-recmask RECOVERYMASK.fits: recovery mask file (required with -mask)\n");
+    fprintf(stderr, "\t-weight WEIGHT.fits: weight file to re-streak\n");
+    fprintf(stderr, "\t-recweight RECOVERYWEIGHT.fits: recovery weight file (required with -weight)\n");
+    fprintf(stderr, "\t-replace: replace the input images with the output\n");
+    fprintf(stderr, "\t-remove: remove the original image after processing (requires -replace)\n");
+
+    exit(2);
+}
+
+static pmConfig *parseArguments(int argc, char **argv)
+{
+    pmConfig *config = pmConfigRead(&argc, argv, NULL);
+    if (!config) {
+        streaksExit("failed to parse arguments", PS_EXIT_CONFIG_ERROR);
+    }
+
+    int argnum;
+    ippStage stage = IPP_STAGE_NONE;
+    if ((argnum = psArgumentGet(argc, argv, "-stage"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        stage = parseStage(argv[argnum]);
+        psMetadataAddS32(config->arguments, PS_LIST_TAIL, "STAGE", 0,
+                "pipeline stage for streak removal", stage);
+        psArgumentRemove(argnum, &argc, argv);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-stage is required\n");
+        usage();
+    }
+
+    // If we are asked to replace inputs with the outputs and input image is in nebulous
+    // we require mask, weight, and tmproot to all be
+    bool nebulousImage = false;
+    if ((argnum = psArgumentGet(argc, argv, "-image"))) {
+        // for raw and chip level images we use psFits for I/O and ...
+        nebulousImage = IN_NEBULOUS(argv[argnum+1]);
+        if (CHIP_LEVEL_INPUT(stage)) {
+            psArgumentRemove(argnum, &argc, argv);
+            psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT", 0,
+                    "name of input image", argv[argnum]);
+            psArgumentRemove(argnum, &argc, argv);
+        } else  {
+            // ... for warped images we use pmFPAfiles
+            if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "INPUT", "-image", NULL)) { ;
+                psError(PS_ERR_UNKNOWN, false, "failed to process -image");
+                usage();
+            }
+        }
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-image is required\n");
+        usage();
+    }
+
+    bool gotReplace = false;
+    if ((argnum = psArgumentGet(argc, argv, "-replace"))) {
+        gotReplace = true;
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files",
+            true);
+    }
+    
+    bool gotMask = false;
+    if ((argnum = psArgumentGet(argc, argv, "-mask"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        bool nebulousMask = IN_NEBULOUS(argv[argnum]);
+        if (nebulousMask != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            usage();
+        }
+        gotMask = true;
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0,
+                "name of input mask image", argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    }
+
+    bool gotWeight = false;
+    if ((argnum = psArgumentGet(argc, argv, "-weight"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        bool nebulousWeight = IN_NEBULOUS(argv[argnum]);
+        if (nebulousWeight != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            usage();
+        }
+        gotWeight = true;
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0,
+                "name of input weight image", argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    }
+
+    if ((argnum = psArgumentGet(argc, argv, "-tmproot"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        bool nebulousTmp = IN_NEBULOUS(argv[argnum]);
+        if (gotReplace && (nebulousTmp != nebulousImage)) {
+            psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path with -replace\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            usage();
+        }
+        psString dir = pathToDirectory(argv[argnum]);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files",
+            dir);
+        psArgumentRemove(argnum, &argc, argv);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-tmproot is required\n");
+        usage();
+    }
+
+    if ((argnum = psArgumentGet(argc, argv, "-recimage"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.IMAGE", 0, "recovery image file name",
+            argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-recimage is required\n");
+        usage();
+    }
+    if ((argnum = psArgumentGet(argc, argv, "-recmask"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        bool nebulousMask = IN_NEBULOUS(argv[argnum]);
+        if (nebulousMask != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            usage();
+        }
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0,
+                "name of input mask image", argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    } else if (gotMask) {
+        psError(PS_ERR_UNKNOWN, true, "-recmask is required with -mask\n");
+        usage();
+    }
+
+    if ((argnum = psArgumentGet(argc, argv, "-recweight"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        bool nebulousWeight = IN_NEBULOUS(argv[argnum]);
+        if (nebulousWeight != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            usage();
+        }
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0,
+                "name of input weight image", argv[argnum]);
+        psArgumentRemove(argnum, &argc, argv);
+    } else if (gotWeight) {
+        psError(PS_ERR_UNKNOWN, true, "-recweight is required with -weight\n");
+        usage();
+    }
+
+    if ((argnum = psArgumentGet(argc, argv, "-remove"))) {
+        if (!gotReplace) {
+            psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n");
+            usage();
+        }
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files",
+            true);
+    }
+
+    if (argc != 1) {
+        psString unexpectedArguments = NULL;
+        for (int i=1; i<argc; i++) {
+            psStringAppend(&unexpectedArguments, "%s ", argv[i]);
+        }
+        psError(PS_ERR_UNKNOWN, true, "unexpected arguments: %s", unexpectedArguments);
+        psFree(unexpectedArguments);
+        usage();
+    }
+
+    return config;
+}
+
+// set the image for the output files to the contents of the corresponding input file.
+static bool readAndCopyToOutput(streakFiles *sf, bool restoreImageCube)
+{
+    if (sf->inImage->pmfile) {
+        // image data from pmFPAfile (diff or warp file)
+        readImageFrom_pmFile(sf);
+    } else {
+        // image data directly from psFits
+        readImage(sf->inImage, sf->extnum, sf->stage);
+    }
+    // read the recovery pixels
+    readImage(sf->recImage, sf->extnum, sf->stage);
+
+    sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
+
+    if (!SFILE_IS_IMAGE(sf->inImage)) {
+        // fprintf(stderr, "skipping extnum: %d not an image\n", sf->extnum);
+        // XXX: This extension is probably a video table.
+        copyTable(sf->outImage, sf->inImage, sf->extnum);
+        return false;
+    }
+
+    if (sf->inImage->image) {
+        setupImageRefs(sf->outImage, NULL, sf->inImage, sf->extnum, false);
+    }  else {
+        // Image cubes are handeled specially
+    }
+
+    // set up the compression parameters
+#ifdef STREAKS_COMPRESS_OUTPUT
+    // compression of the image pixels is disabled for now. Some consortium members
+    // have problems reading compressed images.
+    copyFitsOptions(sf->outImage, sf->recImage, sf->inImage);
+
+    // XXX: TODO: can we derive these values from the input header?
+    // psFitsCompressionGet(sf->inImage->image) gives compression none
+    // perhaps we should just use the definition of COMP_IMG in the configuration 
+    psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
+#endif
+    if (sf->inMask) {
+        readImage(sf->inMask, sf->extnum, sf->stage);
+        readImage(sf->recMask, sf->extnum, sf->stage);
+        sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+
+        // XXX: TODO
+        setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
+
+        // XXX: see note above
+        copyFitsOptions(sf->outMask, NULL, sf->inMask);
+        psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+    }
+
+    if (sf->inWeight) {
+        readImage(sf->inWeight, sf->extnum, sf->stage);
+        readImage(sf->recWeight, sf->extnum, sf->stage);
+
+        sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
+        setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
+
+        copyFitsOptions(sf->outWeight, NULL, sf->inWeight);
+        // XXX: see note above
+        psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
+    }
+
+    return true;
+}
+
+static void writeImages(streakFiles *sf, bool exciseImageCube)
+{
+    psString extname = NULL;
+    if (sf->nHDU > 1) {
+        bool mdok;
+        extname = psMetadataLookupStr(&mdok, sf->inImage->header, "EXTNAME");
+    }
+    if (sf->inImage->numZPlanes == 0)  {
+        // note exciseing complete images is handled in readAndCopyToOutput
+        writeImage(sf->outImage, extname, sf->extnum);
+    } else {
+#ifdef TODO
+        // we have an image cube
+        double initValue;
+        // otherwise write it to the output 
+        writeImageCube(sf->outImage, sf->recImage->imagecube, extname, sf->extnum);
+
+        // borrow one of the images from the imagecube and set it to init value
+        psImage *image = psArrayGet (sf->inImage->imagecube, 0);
+        psMemIncrRefCounter(image);
+        psImageInit(image, initValue);
+        if (exciseImageCube) {
+            sf->outImage->image = image;
+            writeImage(sf->outImage, extname, sf->extnum);
+        } else {
+            // write zero valued image to reccovery
+            if (sf->recImage) {
+                sf->recImage->image = image;
+                writeImage(sf->recImage, extname, sf->extnum);
+            }
+        }
+#endif
+        // Assumption: there are no mask and weight images for video cells because they only appear
+        // at the raw stage. TODO: insure this
+        return;
+    }
+    if (sf->outMask) {
+        writeImage(sf->outMask, extname, sf->extnum);
+    }
+    if (sf->outWeight) {
+        writeImage(sf->outWeight, extname, sf->extnum);
+    }
+}
+
+static bool replicateOutputs(streakFiles *sfiles)
+{
+    bool status = false;
+
+    // XXX: TODO: need a nebGetXatrr function, but there isn't one
+    // another option would be to take the number of copies to be
+    // created as an option. That way the system could decide
+    // whether to replicate anything other than raw Image files
+    void *xattr = NULL;
+
+    if (!replicate(sfiles->outImage, xattr)) {
+        psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+        return false;
+    }
+
+#ifdef notyet
+    // XXX: don't replicate mask and weight images until we can look up
+    // the input's xattr. There may be a perl program that can getXattr
+    if (sfiles->outMask) {
+        // get xattr from input to see if we need to replicate
+        if (!replicate(sfiles->outMask, xattr)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+    if (sfiles->outWeight) {
+        // get xattr from input to see if we need to replicate
+        if (!replicate(sfiles->outWeight, xattr)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+#endif
+
+//      replicate the recovery images (if in nebulous)
+//      perhaps whether we do that or not should be configurable.
+//      Sounds like we need a recipe
+
+    return true;
+}
+
Index: branches/bills_081204/magic/remove/src/streaksutil.c
===================================================================
--- branches/bills_081204/magic/remove/src/streaksutil.c	(revision 20890)
+++ branches/bills_081204/magic/remove/src/streaksutil.c	(revision 20890)
@@ -0,0 +1,41 @@
+#include "streaksremove.h"
+
+psString pathToDirectory(char *path)
+{
+    // insure that path ends in /
+    int len = strlen(path);
+    psString dir = NULL;
+    if (strrchr(path, '/') != (path + len - 1)) {
+        psStringAppend(&dir, "%s/", path);
+    } else {
+        dir = psStringCopy(path);
+    }
+    return dir;
+}
+
+ippStage
+parseStage(psString stageStr)
+{
+    if (!strcmp("raw", stageStr)) {
+        return IPP_STAGE_RAW;
+    } else if (!strcmp("chip", stageStr)) {
+        return IPP_STAGE_CHIP;
+    } else if (!strcmp("warp", stageStr)) {
+        return IPP_STAGE_WARP;
+    } else if (!strcmp("diff", stageStr)) {
+        return IPP_STAGE_DIFF;
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "unknown stage string: %s\n", stageStr);
+        streaksExit("", PS_EXIT_DATA_ERROR);
+        // notreached
+        return IPP_STAGE_NONE;
+    }
+}
+
+// to enhance clarity in these programs we don't propagate errors up the stack
+// we just bail out
+void streaksExit(psString str, int exitCode) {
+    psErrorStackPrint(stderr, str);
+    exit(exitCode);
+}
+
Index: branches/bills_081204/magic/remove/src/warpedpixels.c
===================================================================
--- branches/cnb_branch_20081104/magic/remove/src/warpedpixels.c	(revision 20530)
+++ branches/bills_081204/magic/remove/src/warpedpixels.c	(revision 20890)
@@ -1,4 +1,6 @@
-#include "streaks.h"
+#include "streaksremove.h"
 #include "pmAstrometryWCS.h"
+
+// #define DEBUG_PRINT 1
 
 static void addTouchedPixels(streakFiles *sf, psString fileName);
@@ -7,8 +9,17 @@
 static int xRight(psPlane *pt, int y);
 
-void
+bool
 computeWarpedPixels(streakFiles *sf)
 {
+    bool status;
     pmConfig    *config = sf->config;
+
+    psArray *skycells = psMetadataLookupPtr(&status, config->arguments, "SKYCELLS");
+    if (skycells == NULL) {
+        return false;
+    }
+
+    int n = psArrayLength(skycells);
+
     psRegion     bounds = *pmChipPixels(sf->chip);
     
@@ -19,9 +30,4 @@
     psImageInit(sf->warpedPixels, 0);
 
-    bool status;
-    psArray *skycells = psMetadataLookupPtr(&status, config->arguments, "SKYCELLS");
-
-    int n = psArrayLength(skycells);
-
     for (int i=0; i<n; i++) {
         psString filename = psArrayGet(skycells, i);
@@ -31,9 +37,15 @@
     }
 
-    psFits *fits = psFitsOpen("warpedpixels.fits", "w");
-    psFitsWriteImage(fits, NULL, sf->warpedPixels, 0, NULL);
-    psFitsClose(fits);
-
-    exit (0);
+    bool writeTouchedPixels = false; // XXX: make this an argument to the program
+    if (writeTouchedPixels) {
+        // write out the warped pixels
+        psMetadata * header = psMetadataAlloc();
+        pmAstromWriteWCS(header, sf->inAstrom->fpa, sf->chip, 0.001);
+        psFits *fits = psFitsOpen("warpedpixels.fits", "w");
+
+        psFitsWriteImage(fits, header, sf->warpedPixels, 0, NULL);
+        psFitsClose(fits);
+    }
+    return true;
 }
 
@@ -45,10 +57,10 @@
     if (!fits) {
         psError(PS_ERR_IO, false, "failed to open skycell file: %s", fileName);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
+        streaksExit("", PS_EXIT_DATA_ERROR);
     }
     psMetadata *header = psFitsReadHeader(NULL, fits);
     if (!header) {
         psError(PS_ERR_IO, false, "failed to read fixts header from skycell file: %s", fileName);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
+        streaksExit("", PS_EXIT_DATA_ERROR);
     }
     // set up astrometry for conversion from skycell image to sky
@@ -56,5 +68,5 @@
     if (!wcs) {
         psError(PS_ERR_IO, false, "failed to read astrometry  from skycell file: %s", fileName);
-        streaksremoveExit("", PS_EXIT_DATA_ERROR);
+        streaksExit("", PS_EXIT_DATA_ERROR);
     }
     int naxis1 = psMetadataLookupS32(NULL, header, "NAXIS1");
@@ -62,5 +74,5 @@
 
     /* now set up our wrapper to the chip astrometry to apply to the whole chip */
-    sf->astrom = streakSetAstrometry(sf->astrom, sf->inAstrom->fpa, sf->chip, false, NULL, 
+    sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, sf->inAstrom->fpa, sf->chip, false, NULL, 
         sf->warpedPixels->numCols, sf->warpedPixels->numRows);
     
@@ -87,5 +99,5 @@
         if (!pmAstromWCStoSky(&sky, wcs, &pt[i])) {
             psError(PS_ERR_IO, false, "failed to convert pt %d of %s to sky coords: %s", fileName);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
+            streaksExit("", PS_EXIT_DATA_ERROR);
         }
         strkPt p;
@@ -93,5 +105,5 @@
         if (!skyToCell(&p, sf->astrom, sky.r, sky.d)) {
             psError(PS_ERR_IO, false, "failed to convert pt %d of %s to sky coords: %s", fileName);
-            streaksremoveExit("", PS_EXIT_DATA_ERROR);
+            streaksExit("", PS_EXIT_DATA_ERROR);
         }
         pt[i].x = p.x;
@@ -108,14 +120,17 @@
         type = "2";
     }
+#ifdef DEBUG_PRINT
     printf("\nSKYCELL %s Type: %s\n", fileName, type);
     for (int i=0; i<4; i++) {
         printf("%f %f\n", pt[i].x, pt[i].y);
     }
-
+#endif
     // Now set the touched pixels
-    int ymin = fmax(0, pt[1].y);
-    int ymax = fmin(pt[3].y, sf->warpedPixels->numRows);
+    int ymin = fmax(0, pt[1].y );
+    int ymax = fmin(pt[3].y + 0.5, sf->warpedPixels->numRows - 1);
+#if (DEBUG_PRINT > 1)
     printf("\nymin: %d ymax: %d\n", ymin, ymax);
-    for (int y = ymin ; y < ymax; y++) {
+#endif
+    for (int y = ymin ; y <= ymax; y++) {
         int xleft  = xLeft(pt, y);
         int xright = xRight(pt, y);
@@ -125,18 +140,21 @@
         }
         if (xleft > sf->warpedPixels->numCols) {
-            xleft = sf->warpedPixels->numCols - 1;
+            continue;
         }
         if (xright < 0) {
-            xright = 0;
+            continue;
         }
         if (xright >= sf->warpedPixels->numCols) {
             xright = sf->warpedPixels->numCols - 1;
         }
+#if (DEBUG_PRINT > 1)
         printf("  y: %d xleft: %d xright: %d\n", y, xleft, xright);
-
-        psU8 *scanline = sf->warpedPixels->data.U8[y];
-
-        for (int x = xleft ; x <= xright; x++) {
-            scanline[x] = 1;
+#endif
+
+        psU8 *rowPixels = sf->warpedPixels->data.U8[y];
+
+        int n = xright - xleft + 1;
+        if (n > 0) {
+            memset(&rowPixels[xleft], 255, n);
         }
     }
@@ -164,5 +182,5 @@
         x_d = xOfY(&pt[0], &pt[3], y);
     }
-    return (int) x_d;
+    return (int) (x_d + 0.5);
 }
 
@@ -253,8 +271,8 @@
     /* find pt0 the left most (bottom most) corner */
     int imin = 0;
-    double  min = pt[0].x;
+    int  min = (int) pt[0].x;
     int i;
-    for (i=0; i<4; i++) {
-        double x = pt[i].x;
+    for (i=1; i<4; i++) {
+        int x = pt[i].x;
         if ((x < min) ||
             ((x == min) && (pt[i].y < pt[imin].y))) {
@@ -272,6 +290,6 @@
     imin = 0;
     min = pt[0].y;
-    for (i=0; i<3; i++) {
-        double y = pt[i].y;
+    for (i=1; i<3; i++) {
+        int y = pt[i].y;
         if ((y < min) ||
             ((y == min) && (pt[i].x > pt[imin].x)) ) {
Index: branches/bills_081204/ppConfigDump/src/ppConfigDump.c
===================================================================
--- branches/cnb_branch_20081104/ppConfigDump/src/ppConfigDump.c	(revision 20530)
+++ branches/bills_081204/ppConfigDump/src/ppConfigDump.c	(revision 20890)
@@ -103,5 +103,5 @@
         }
 
-        psMetadata *format = pmConfigCameraFormatFromHeader(NULL, NULL, config, phu, true); // Camera format
+        psMetadata *format = pmConfigCameraFormatFromHeader(NULL, NULL, NULL, config, phu, true); // Camera format
         if (!format || !config->camera) {
             psErrorStackPrint(stderr, "Can't find suitable camera configuration!\n");
Index: branches/bills_081204/ppImage/src/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/Makefile.am	(revision 20530)
+++ branches/bills_081204/ppImage/src/Makefile.am	(revision 20890)
@@ -17,4 +17,5 @@
 	ppImageDetrendNonLinear.c \
 	ppImageDetrendFringe.c \
+	ppImageDetrendFree.c \
 	ppImageRebinReadout.c \
 	ppImageMosaic.c \
Index: branches/bills_081204/ppImage/src/ppImage.h
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImage.h	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImage.h	(revision 20890)
@@ -31,4 +31,5 @@
     bool doBias;                        // Bias subtraction
     bool doDark;                        // Dark subtraction
+    bool doRemnance;                    // Remnance masking
     bool doShutter;                     // Shutter correction
     bool doFlat;                        // Flat-field normalisation
@@ -83,4 +84,6 @@
     float fringeKeep;                   // Fringe keep fraction
 
+    int remnanceSize;                   // Size for remnance detection
+    float remnanceThresh;               // Threshold for remnance detection
 } ppImageOptions;
 
@@ -125,4 +128,7 @@
 pmReadout* ppImageDetrendSelectFirst(pmCell *cell, char *name, bool doThis);
 
+bool ppImageDetrendFree(pmConfig *config, pmFPAview *view);
+bool ppImageFringeFree(pmConfig *config, pmFPAview *view);
+
 // Record which detrend file was used for the detrending
 bool ppImageDetrendRecord(
Index: branches/bills_081204/ppImage/src/ppImageDetrendFree.c
===================================================================
--- branches/bills_081204/ppImage/src/ppImageDetrendFree.c	(revision 20890)
+++ branches/bills_081204/ppImage/src/ppImageDetrendFree.c	(revision 20890)
@@ -0,0 +1,69 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "ppImage.h"
+
+// list of the detrend types to free
+static char *detrendTypes[] = {
+    "PPIMAGE.MASK",
+    "PPIMAGE.BIAS",
+    "PPIMAGE.DARK",
+    "PPIMAGE.FLAT",
+    "PPIMAGE.SHUTTER",
+    NULL
+};
+
+bool ppImageDetrendFree (pmConfig *config, pmFPAview *view) {
+
+    bool status;
+
+    for (int i = 0; detrendTypes[i] != NULL; i++) {
+
+	pmFPAfile *file = psMetadataLookupPtr(&status, config->files, detrendTypes[i]); // File of interest
+	if (!file) continue; // not all detrends are used in any given run
+
+	// this only returns false on a failure.  if we are not ready to write or close, it is not an error
+	if (!pmFPAfileWrite (file, view, config)) {
+	    psError(PS_ERR_IO, false, "failed to WRITE %s", file->name);
+	    return false;
+	}
+	if (!pmFPAfileClose(file, view)) {
+	    psError(PS_ERR_IO, false, "failed to CLOSE for %s", file->name);
+	    return false;
+	}
+	if (!pmFPAfileFreeData(file, view)) {
+	    if (!psMetadataRemoveKey(config->files, file->name)) {
+		psError(PS_ERR_IO, false, "failed to remove %s in FPA_AFTER block", file->name);
+		return false;
+	    }
+	}
+    }
+    return true;
+}
+
+bool ppImageFringeFree (pmConfig *config, pmFPAview *view) {
+
+    bool status;
+
+    pmFPAfile *file = psMetadataLookupPtr(&status, config->files, "PPIMAGE.FRINGE"); // File of interest
+    if (!file) return true;
+
+    // this only returns false on a failure.  if we are not ready to write or close, it is not an error
+    if (!pmFPAfileWrite (file, view, config)) {
+	psError(PS_ERR_IO, false, "failed to WRITE %s", file->name);
+	return false;
+    }
+    if (!pmFPAfileClose(file, view)) {
+	psError(PS_ERR_IO, false, "failed to CLOSE for %s", file->name);
+	return false;
+    }
+    if (!pmFPAfileFreeData(file, view)) {
+	if (!psMetadataRemoveKey(config->files, file->name)) {
+	    psError(PS_ERR_IO, false, "failed to remove %s in FPA_AFTER block", file->name);
+	    return false;
+	}
+    }
+
+    return true;
+}
Index: branches/bills_081204/ppImage/src/ppImageDetrendReadout.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageDetrendReadout.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageDetrendReadout.c	(revision 20890)
@@ -71,4 +71,12 @@
     }
 
+    if (options->doRemnance) {
+        if (!pmRemnance(input, options->maskValue, options->badMask,
+                        options->remnanceSize, options->remnanceThresh)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to mask remnance.");
+            return false;
+        }
+    }
+
     // Shutter correction
     if (options->doShutter) {
Index: branches/bills_081204/ppImage/src/ppImageDetrendRecord.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageDetrendRecord.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageDetrendRecord.c	(revision 20890)
@@ -66,4 +66,5 @@
     psMetadata *detrend = psMetadataAlloc(); // Detrend information
     psMetadataAddMetadata(cell->analysis, PS_LIST_TAIL, "DETREND", 0, "Detrend information", detrend);
+
     detrendRecord(options->doMask, detrend, config, view, "PPIMAGE.MASK", "DETREND.MASK", "Mask filename");
     detrendRecord(options->doBias, detrend, config, view, "PPIMAGE.BIAS", "DETREND.BIAS", "Bias filename");
@@ -74,4 +75,6 @@
     detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE",
                   "Fringe filename");
+
+    psFree (detrend);
     return true;
 }
Index: branches/bills_081204/ppImage/src/ppImageLoop.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageLoop.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageLoop.c	(revision 20890)
@@ -95,4 +95,9 @@
                     ESCAPE("Unable to detrend readout");
                 }
+
+		// free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+		if (!ppImageDetrendFree (config, view)) {
+		    ESCAPE("Unable to free detrend images");
+		}
             }
 
@@ -100,5 +105,13 @@
                 ppImageDetrendRecord(cell, config, options, view);
             }
-        }
+	    // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+	    if (!ppImageDetrendFree (config, view)) {
+		ESCAPE("Unable to free detrend images");
+	    }
+        }
+	// free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+	if (!ppImageDetrendFree (config, view)) {
+	    ESCAPE("Unable to free detrend images");
+	}
 
         // Apply the fringe correction
@@ -108,4 +121,8 @@
             }
         }
+	// free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+	if (!ppImageFringeFree (config, view)) {
+	    ESCAPE("Unable to free fringe images");
+	}
 
         // measure various statistics for this image
@@ -134,4 +151,16 @@
             }
         }
+
+	// these may be used by ppImageSubtractBackground.
+	// if these are defined as internal files, drop them here
+	status = true;
+	status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL");
+	status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV");
+	status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND");
+	if (!status) {
+	    psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files");
+	    psFree (view);
+	    return false;
+	}
 
         // binning (used for display) must take place after the background is replaced, if desired
Index: branches/bills_081204/ppImage/src/ppImageOptions.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageOptions.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageOptions.c	(revision 20890)
@@ -25,4 +25,5 @@
     options->doBias          = false;   // Bias subtraction
     options->doDark          = false;   // Dark subtraction
+    options->doRemnance      = false;   // Remnance masking
     options->doShutter       = false;   // Shutter correction
     options->doFlat          = false;   // Flat-field normalisation
@@ -75,4 +76,8 @@
     options->fringeIter      = 0;       // Fringe iterations
     options->fringeKeep      = 1.0;     // Fringe keep fraction
+
+    // Remnance values
+    options->remnanceSize    = 30;      // Size for remnance detection
+    options->remnanceThresh  = 25.0;    // Threshold for remnance detection
 
     return options;
@@ -196,4 +201,5 @@
     options->doBias = psMetadataLookupBool(NULL, recipe, "BIAS");
     options->doDark = psMetadataLookupBool(NULL, recipe, "DARK");
+    options->doRemnance = psMetadataLookupBool(NULL, recipe, "REMNANCE");
     options->doFlat = psMetadataLookupBool(NULL, recipe, "FLAT");
     options->doFringe = psMetadataLookupBool(NULL, recipe, "FRINGE");
@@ -268,4 +274,7 @@
     options->fringeKeep = psMetadataLookupF32(NULL, recipe, "FRINGE.KEEP");
 
+    // Remnance options
+    options->remnanceSize = psMetadataLookupF32(NULL, recipe, "REMNANCE.SIZE");
+    options->remnanceThresh = psMetadataLookupS32(NULL, recipe, "REMNANCE.THRESH");
 
     return options;
Index: branches/bills_081204/ppImage/src/ppImageParseCamera.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageParseCamera.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageParseCamera.c	(revision 20890)
@@ -298,5 +298,5 @@
         PS_ASSERT (psphotOutput, false);
 
-        pmFPAfile *psastroInput = pmFPAfileDefineInput (config, psphotOutput->fpa, "PSASTRO.INPUT");
+        pmFPAfile *psastroInput = pmFPAfileDefineInput (config, psphotOutput->fpa, NULL, "PSASTRO.INPUT");
         PS_ASSERT (psastroInput, false);
         psastroInput->mode = PM_FPA_MODE_REFERENCE;
Index: branches/bills_081204/ppImage/src/ppImageReplaceBackground.c
===================================================================
--- branches/cnb_branch_20081104/ppImage/src/ppImageReplaceBackground.c	(revision 20530)
+++ branches/bills_081204/ppImage/src/ppImageReplaceBackground.c	(revision 20890)
@@ -20,12 +20,4 @@
     if (!status) {
         psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined");
-        return false;
-    }
-
-    // The background model file may be defined, though the model hasn't been built
-    // If so, we will build it below.
-    pmFPAfile *modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model
-    if (!status) {
-        psError(PS_ERR_UNEXPECTED_NULL, true, "PSPHOT.BACKMDL file is not defined");
         return false;
     }
@@ -75,16 +67,33 @@
     psImage *image = ro->image, *mask = ro->mask; // Image and mask of interest
 
+    // View corresponding to this readout
+    pmFPAview roView = *view;
+    roView.cell = roView.readout = 0;
 
-    pmFPAview roView = *view;           // View to readout
-    roView.cell = roView.readout = 0;
-    pmReadout *modelRO = pmFPAviewThisReadout(&roView, modelFile->fpa); // Background model
-    if (!modelRO) {
-        // Create the background model
+    // If the background model file has not been defined, psphotModelBackground will generate it
+    pmReadout *modelRO = NULL;
+    pmFPAfile *modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model
+    if (modelFile) {
+        modelRO = pmFPAviewThisReadout(&roView, modelFile->fpa); // Background model
+    }
+
+    // the background model has not been defined, or at least not generated
+    if (!modelFile || !modelRO) {
         if (!psphotModelBackground(config, &roView, "PPIMAGE.CHIP")) {
             psError(PS_ERR_UNKNOWN, false, "Unable to model background");
             return false;
         }
-        modelRO = (modelFile->mode == PM_FPA_MODE_INTERNAL) ? modelFile->readout :
-                   pmFPAviewThisReadout(&roView, modelFile->fpa);
+        // the model file should now at least be defined
+        modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model
+        if (!modelFile) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to define background model I/O file");
+            return false;
+        }
+        // now grab the readout from the correct location:
+        if (modelFile->mode == PM_FPA_MODE_INTERNAL) {
+            modelRO = modelFile->readout;
+        } else {
+            modelRO = pmFPAviewThisReadout(&roView, modelFile->fpa);
+        }
         if (!modelRO) {
             psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background model");
@@ -92,8 +101,56 @@
         }
     }
-    psImageBinning *binning = psMetadataLookupPtr(&status, psphotRecipe,
-                                                  "PSPHOT.BACKGROUND.BINNING"); // Binning for model
-    psImage *modelImage = modelRO->image; // Background model
+    psImageBinning *binning = psMetadataLookupPtr(&status, psphotRecipe, "PSPHOT.BACKGROUND.BINNING"); // Binning for model
+    if (!binning) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background binning");
+        return false;
+    }
 
+# define USE_UNBIN 1
+# if (USE_UNBIN)
+    // select background pixels, from output background file, or create
+    pmReadout *background = NULL;
+    pmFPAfile *backfile = psMetadataLookupPtr (&status, config->files, "PSPHOT.BACKGND");
+    if (backfile) {
+        // we are using PSPHOT.BACKGND as an I/O file: select readout or create
+        if (backfile->mode == PM_FPA_MODE_INTERNAL) {
+            background = backfile->readout;
+        } else {
+            background = pmFPAviewThisReadout (&roView, backfile->fpa);
+        }
+        if (background == NULL) {
+            // readout does not yet exist: create from input
+            pmFPAfileCopyStructureView (backfile->fpa, input->fpa, 1, 1, &roView);
+            background = pmFPAviewThisReadout (&roView, backfile->fpa);
+            if ((image->numCols != background->image->numCols) || (image->numRows != background->image->numRows)) {
+                psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for background dimensions");
+                return false;
+            }
+        }
+    } else {
+        background = pmFPAfileDefineInternal (config->files, "PSPHOT.BACKGND", image->numCols, image->numRows, PS_TYPE_F32);
+    }
+    psF32 **backData = background->image->data.F32;
+
+    // linear interpolation to full-scale
+    if (!psImageUnbin (background->image, modelRO->image, binning)) {
+        psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning");
+        return false;
+    }
+
+    // Do the background subtraction
+    int numCols = image->numCols, numRows = image->numRows; // Size of image
+    for (int y = 0; y < numRows; y++) {
+        for (int x = 0; x < numCols; x++) {
+            float value = backData[y][x];
+            if (!isfinite(value)) {
+                image->data.F32[y][x] = NAN;
+                mask->data.PS_TYPE_MASK_DATA[y][x] |= options->badMask;
+            } else {
+                image->data.F32[y][x] -= value;
+            }
+        }
+    }
+# else
     // Do the background subtraction
     int numCols = image->numCols, numRows = image->numRows; // Size of image
@@ -103,5 +160,5 @@
                 image->data.F32[y][x] = 0.0;
             } else {
-                float value = psImageUnbinPixel_V2(x, y, modelImage, binning); // Background value
+                float value = psImageUnbinPixel(x + 0.5, y + 0.5, modelRO->image, binning); // Background value
                 if (!isfinite(value)) {
                     image->data.F32[y][x] = NAN;
@@ -113,7 +170,9 @@
         }
     }
+# endif
 
-    pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL");
-    pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL.STDEV");
+    // XXX should these really be here?? (probably not...)
+    // pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL");
+    // pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL.STDEV");
 
     return true;
Index: branches/bills_081204/ppMerge/src/ppMergeFiles.c
===================================================================
--- branches/cnb_branch_20081104/ppMerge/src/ppMergeFiles.c	(revision 20530)
+++ branches/bills_081204/ppMerge/src/ppMergeFiles.c	(revision 20890)
@@ -58,4 +58,11 @@
         }
         psFree(fileView);
+
+        // Read the headers, so we can have the concepts available
+        pmCell *cell = pmFPAviewThisCell(view, input->fpa); // Cell of interest
+        if (!pmCellReadHeaderSet(cell, input->fits, config)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to read headers for image %d", num);
+            return false;
+        }
     }
     bool mdok;          // Status of MD lookup
Index: branches/bills_081204/ppStack/src/ppStack.h
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStack.h	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStack.h	(revision 20890)
@@ -11,6 +11,7 @@
 typedef enum {
     PPSTACK_MASK_MATCH  = 0x01,         // PSF-matching failed
-    PPSTACK_MASK_REJECT = 0x02,         // Rejection failed
-    PPSTACK_MASK_BAD    = 0x04,         // Bad image (too many pixels rejected)
+    PPSTACK_MASK_CHI2   = 0x02,         // Chi^2 too deviant
+    PPSTACK_MASK_REJECT = 0x04,         // Rejection failed
+    PPSTACK_MASK_BAD    = 0x08,         // Bad image (too many pixels rejected)
     PPSTACK_MASK_ALL    = 0xff          // All errors
 } ppStackMask;
@@ -98,4 +99,5 @@
                                const psArray *regions, // Array with array of regions used in each PSF match
                                const psArray *kernels, // Array with array of kernels used in each PSF match
+                               const psVector *weightings, // Weighting factors for each image
                                const psVector *addVariance // Additional variance for rejection
     );
@@ -113,5 +115,6 @@
                          pmReadout *outRO,   // Output readout
                          const psArray *readouts, // Input readouts
-                         const psArray *rejected // Array with pixels rejected in each image
+                         const psArray *rejected, // Array with pixels rejected in each image
+                         const psVector *weightings // Weighting factors for each image
     );
 
@@ -141,5 +144,6 @@
                   psArray **kernels,    // Array of kernels used in each PSF matching, returned
                   float *chi2,          // Chi^2 from the stamps
-                  const psArray *sources, // Array of sources
+                  float *weighting,     // Stack weighting (1/noise^2)
+                  psArray *sources,     // Array of sources
                   const pmPSF *psf,     // Target PSF
                   psRandom *rng,        // Random number generator
Index: branches/bills_081204/ppStack/src/ppStackCamera.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackCamera.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackCamera.c	(revision 20890)
@@ -101,10 +101,4 @@
         psString psf = psMetadataLookupStr(&mdok, input, "PSF"); // Name of PSF
         psString sources = psMetadataLookupStr(&mdok, input, "SOURCES"); // Name of sources
-
-        float weighting = psMetadataLookupF32(&mdok, input, "WEIGHTING"); // Relative weighting
-        if (!mdok || isnan(weighting) || weighting == 0.0) {
-            psWarning("Component %s lacks WEIGHTING of type F32 --- assuming 1.0", item->name);
-            weighting = 1.0;
-        }
 
         // Add the image file
@@ -258,7 +252,4 @@
 #endif
 
-        psMetadataAddF32(imageFile->fpa->analysis, PS_LIST_TAIL, "PPSTACK.WEIGHTING", 0,
-                         "Relative weighting for image", weighting);
-
         i++;
     }
Index: branches/bills_081204/ppStack/src/ppStackLoop.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackLoop.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackLoop.c	(revision 20890)
@@ -346,5 +346,10 @@
 
                     psFree(view);
-                    filesIterateUp(config);
+                    if (!filesIterateUp(config)) {
+                        psFree(globalSources);
+                        psFree(indSources);
+                        psFree(targetPSF);
+                        return false;
+                    }
                     pmFPAfileActivate(config->files, false, NULL);
                     fileActivation(config, prepareFiles, true);
@@ -411,5 +416,7 @@
 
         psFree(view);
-        filesIterateUp(config);
+        if (!filesIterateUp(config)) {
+            return false;
+        }
     }
 
@@ -444,4 +451,6 @@
     psVector *matchChi2 = psVectorAlloc(num, PS_TYPE_F32); // chi^2 for stamps when matching
     psVectorInit(matchChi2, NAN);
+    psVector *weightings = psVectorAlloc(num, PS_TYPE_F32); // Combination weightings for images (1/noise^2)
+    psVectorInit(weightings, NAN);
     for (int i = 0; i < num; i++) {
         psTrace("ppStack", 2, "Convolving input %d of %d to target PSF....\n", i, num);
@@ -482,5 +491,5 @@
         psArray *sources = haveSources ? globalSources : indSources->data[i]; // Sources for matching
         psTimerStart("PPSTACK_MATCH");
-        if (!ppStackMatch(readout, &regions, &kernels, &matchChi2->data.F32[i],
+        if (!ppStackMatch(readout, &regions, &kernels, &matchChi2->data.F32[i], &weightings->data.F32[i],
                           sources, targetPSF, rng, config)) {
             psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i);
@@ -516,5 +525,7 @@
 
         cells->data[i] = psMemIncrRefCounter(readout->parent);
-        filesIterateUp(config);
+        if (!filesIterateUp(config)) {
+            return false;
+        }
         numGood++;
 
@@ -532,4 +543,65 @@
         psFree(cells);
         psFree(inputMask);
+        psFree(matchChi2);
+        return false;
+    }
+
+    // Reject images out-of-hand on the basis of their match chi^2
+    {
+        psVector *values = psVectorAllocEmpty(num, PS_TYPE_F32); // Values to sort
+        for (int i = 0; i < num; i++) {
+            if (inputMask->data.PS_TYPE_MASK_DATA[i] & PPSTACK_MASK_ALL) {
+                continue;
+            }
+            values->data.F32[values->n++] = matchChi2->data.F32[i];
+        }
+        assert(values->n == numGood);
+        if (!psVectorSortInPlace(values)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to sort vector.");
+            psFree(subKernels);
+            psFree(subRegions);
+            psFree(cells);
+            psFree(inputMask);
+            psFree(matchChi2);
+            psFree(values);
+            return false;
+        }
+        float median = numGood % 2 ? values->data.F32[numGood / 2] :
+            0.5 * (values->data.F32[numGood / 2 - 1] + values->data.F32[numGood / 2]);
+
+        float rms = 0.74 * (values->data.F32[numGood * 3 / 4] -
+                            values->data.F32[numGood / 4]); // Estimated RMS from interquartile range
+
+        psFree(values);
+
+        float rej = psMetadataLookupF32(NULL, recipe, "MATCH.REJ"); // Rejection threshold (stdevs)
+        if (isfinite(rej)) {
+            float thresh = median + rej * rms; // Threshold for rejection
+            psLogMsg("ppStack", PS_LOG_INFO, "chi^2 rejection threshold = %f + %f * %f = %f",
+                     median, rej, rms, thresh);
+
+            int numRej = 0;                 // Number rejected
+            numGood = 0;                    // Number of good images
+            for (int i = 0; i < num; i++) {
+                if (inputMask->data.PS_TYPE_MASK_DATA[i] & PPSTACK_MASK_ALL) {
+                    continue;
+                }
+                if (matchChi2->data.F32[i] > thresh) {
+                    numRej++;
+                    inputMask->data.PS_TYPE_MASK_DATA[i] |= PPSTACK_MASK_CHI2;
+                    psLogMsg("ppStack", PS_LOG_INFO, "Rejecting image %d because of large matching chi^2: %f",
+                             i, matchChi2->data.F32[i]);
+                } else {
+                    numGood++;
+                }
+            }
+        }
+    }
+
+    if (numGood == 0) {
+        psError(PS_ERR_UNKNOWN, false, "No good images to combine.");
+        psFree(subKernels);
+        psFree(subRegions);
+        psFree(cells);
         psFree(inputMask);
         psFree(matchChi2);
@@ -657,4 +729,5 @@
             psArrayAdd(job->args, 1, subRegions);
             psArrayAdd(job->args, 1, subKernels);
+            psArrayAdd(job->args, 1, weightings);
             psArrayAdd(job->args, 1, matchChi2);
             if (!psThreadJobAddPending(job)) {
@@ -664,4 +737,5 @@
                 psFree(stack);
                 psFree(inputMask);
+                psFree(weightings);
                 psFree(matchChi2);
                 psFree(view);
@@ -697,4 +771,6 @@
         psThreadJob *job;               // Completed job
         while ((job = psThreadJobGetDone())) {
+            psAssert(strcmp(job->type, "PPSTACK_INITIAL_COMBINE") == 0,
+                     "Job has incorrect type: %s", job->type);
             psArray *results = job->results; // Results of job
             for (int i = 0; i < num; i++) {
@@ -754,5 +830,5 @@
         }
 
-        if (!psThreadPoolWait(false)) {
+        if (!psThreadPoolWait(true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to concatenate inspection lists.");
             psFree(subKernels);
@@ -772,6 +848,11 @@
         }
 
+        int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches
+
         // Reject bad pixels
         for (int i = 0; i < num; i++) {
+            if (inputMask->data.U8[i]) {
+                continue;
+            }
             psTimerStart("PPSTACK_REJECT");
 
@@ -791,5 +872,5 @@
 #endif
 
-            psPixels *reject = pmStackReject(inspect->data[i], numCols, numRows, threshold, poorFrac,
+            psPixels *reject = pmStackReject(inspect->data[i], numCols, numRows, threshold, poorFrac, stride,
                                              subRegions->data[i], subKernels->data[i]); // Rejected pixels
 
@@ -829,4 +910,6 @@
                 }
             }
+
+            // Images without a list of rejected pixels (the list may be empty) are rejected completely
             rejected->data[i] = reject;
 
@@ -895,4 +978,5 @@
             psArrayAdd(job->args, 1, outRO);
             psArrayAdd(job->args, 1, rejected);
+            psArrayAdd(job->args, 1, weightings);
             if (!psThreadJobAddPending(job)) {
                 psFree(job);
@@ -930,4 +1014,5 @@
     psTrace("ppStack", 2, "Cleaning up after combination....\n");
 
+#if 0
     // Ensure masked regions really look masked
     {
@@ -938,4 +1023,5 @@
         }
     }
+#endif
 
     // Close up
@@ -1096,7 +1182,4 @@
     ppStackVersionMetadata(hdu->header);
 
-    // Write out the output files
-    filesIterateUp(config);
-
     psFree(outRO);
     psFree(view);
@@ -1119,4 +1202,9 @@
     }
 
+    // Write out the output files
+    if (!filesIterateUp(config)) {
+        return false;
+    }
+
     memDump("finish");
 
Index: branches/bills_081204/ppStack/src/ppStackMatch.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackMatch.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackMatch.c	(revision 20890)
@@ -47,6 +47,112 @@
 #endif
 
-bool ppStackMatch(pmReadout *readout, psArray **regions, psArray **kernels, float *chi2,
-                  const psArray *sources, const pmPSF *psf, psRandom *rng, const pmConfig *config)
+static psArray *stackSourcesFilter(psArray *sources, // Source list to filter
+                                   int exclusion // Exclusion zone, pixels
+    )
+{
+    psAssert(sources && sources->n > 0, "Require array of sources");
+    if (exclusion <= 0) {
+        return psMemIncrRefCounter(sources);
+    }
+
+    int num = sources->n;               // Number of sources
+    psVector *x = psVectorAlloc(num, PS_TYPE_F32), *y = psVectorAlloc(num, PS_TYPE_F32); // Coordinates
+    int numGood = 0;                    // Number of good sources
+    for (int i = 0; i < num; i++) {
+        pmSource *source = sources->data[i]; // Source of interest
+        if (!source) {
+            continue;
+        }
+        x->data.F32[numGood] = source->peak->xf;
+        y->data.F32[numGood] = source->peak->yf;
+        numGood++;
+    }
+    x->n = y->n = numGood;
+
+    psTree *tree = psTreePlant(2, 2, x, y); // kd tree
+
+    psArray *filtered = psArrayAllocEmpty(numGood); // Filtered list of sources
+    psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of source
+    int numFiltered = 0;                // Number of filtered sources
+    for (int i = 0; i < num; i++) {
+        pmSource *source = sources->data[i]; // Source of interest
+        if (!source) {
+            continue;
+        }
+        coords->data.F64[0] = source->peak->xf;
+        coords->data.F64[1] = source->peak->yf;
+
+        long numWithin = psTreeWithin(tree, coords, exclusion); // Number within exclusion zone
+        psTrace("ppStack", 9, "Source at %.0lf,%.0lf has %ld sources in exclusion zone",
+                coords->data.F64[0], coords->data.F64[1], numWithin);
+        if (numWithin == 1) {
+            // Only itself inside the exclusion zone
+            filtered = psArrayAdd(filtered, filtered->n, source);
+        } else {
+            numFiltered++;
+        }
+    }
+    psFree(coords);
+    psFree(tree);
+
+    psLogMsg("ppStack", PS_LOG_INFO, "Filtered out %d of %d sources", numFiltered, numGood);
+
+    return filtered;
+}
+
+#define BG_SIZE 64                      // Large box half-size in which to measure background
+
+// Generate a background model of the readout we're matching
+psImage *stackBackgroundModel(pmReadout *ro, psMaskType maskVal, const psArray *sources, int size)
+{
+    psImage *image = ro->image, *mask = ro->mask; // Image and mask of readout
+    int numCols = image->numCols, numRows = image->numRows; // Size of image
+
+    psImage *bgImage = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Background image
+    psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics for measuring background
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
+
+    for (int i = 0; i < sources->n; i++) {
+        pmSource *source = sources->data[i]; // Source of interest
+        if (!source) {
+            continue;
+        }
+
+        int x = source->peak->xf + 0.5, y = source->peak->yf + 0.5; // Coordinates of source
+
+        int xMin = PS_MAX(x - BG_SIZE, 0), xMax = PS_MIN(x + BG_SIZE, numCols);
+        int yMin = PS_MAX(y - BG_SIZE, 0), yMax = PS_MIN(y + BG_SIZE, numCols);
+
+        psRegion region = psRegionSet(xMin, xMax, yMin, yMax); // Region of interest
+        psImage *subImage = psImageSubset(image, region); // Subimage
+        psImage *subMask = psImageSubset(mask, region); // Subimage mask
+        if (!psImageBackground(stats, NULL, subImage, subMask, maskVal, rng)) {
+            // Can't do anything about it
+            psErrorClear();
+            continue;
+        }
+        psFree(subImage);
+        psFree(subMask);
+
+        float value = stats->robustMedian; // Value to set background to
+
+        int uMin = PS_MAX(x - size, 0), uMax = PS_MIN(x + size, numCols);
+        int vMin = PS_MAX(y - size, 0), vMax = PS_MIN(y + size, numCols);
+
+        for (int v = vMin; v < vMax; v++) {
+            for (int u = uMin; u < uMax; u++) {
+                bgImage->data.F32[v][u] = value;
+            }
+        }
+    }
+
+    psFree(stats);
+    psFree(rng);
+
+    return bgImage;
+}
+
+bool ppStackMatch(pmReadout *readout, psArray **regions, psArray **kernels, float *chi2, float *weighting,
+                  psArray *sources, const pmPSF *psf, psRandom *rng, const pmConfig *config)
 {
     assert(readout);
@@ -151,4 +257,5 @@
             int footprint = psMetadataLookupS32(NULL, ppsub, "STAMP.FOOTPRINT"); // Stamp half-size
             float threshold = psMetadataLookupF32(NULL, ppsub, "STAMP.THRESHOLD"); // Threshold for stmps
+            int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches
             int iter = psMetadataLookupS32(NULL, ppsub, "ITER"); // Rejection iterations
             float rej = psMetadataLookupF32(NULL, ppsub, "REJ"); // Rejection threshold
@@ -186,9 +293,10 @@
                     psErrorClear();
                 } else {
-                    minFlux = 0.1 * bg->robustStdev;
+                    minFlux = bg->robustStdev;
                 }
                 psFree(bg);
             }
 
+#if 0
             float maxMag = -INFINITY;   // Maximum magnitude of sources
             for (int i = 0; i < sources->n; i++) {
@@ -200,9 +308,13 @@
             }
             minFlux = PS_MIN(FAINT_SOURCE_FRAC * powf(10.0, -0.4 * maxMag), minFlux);
+#endif
 
             pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF
 
-            if (!pmReadoutFakeFromSources(fake, readout->image->numCols, readout->image->numRows, sources,
-                                          NULL, NULL, psf, minFlux, 0, false)) {
+            // For the sake of stamps, remove nearby sources
+            psArray *stampSources = stackSourcesFilter(sources, footprint); // Filtered list of sources
+
+            if (!pmReadoutFakeFromSources(fake, readout->image->numCols, readout->image->numRows,
+                                          stampSources, NULL, NULL, psf, minFlux, 0, false)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image with target PSF.");
                 psFree(fake);
@@ -211,4 +323,10 @@
                 return false;
             }
+
+            // Add the background into the target image
+            psImage *bgImage = stackBackgroundModel(readout, maskVal, stampSources, footprint); // Image of background
+            psBinaryOp(fake->image, fake->image, "+", bgImage);
+            psFree(bgImage);
+
 
 #ifdef TESTING
@@ -221,5 +339,28 @@
                 psFitsClose(fits);
             }
-#endif
+            {
+                psString name = NULL;
+                psStringAppend(&name, "real_%03d.fits", numInput);
+                psFits *fits = psFitsOpen(name, "w");
+                psFree(name);
+                psFitsWriteImage(fits, NULL, readout->image, 0, NULL);
+                psFitsClose(fits);
+            }
+#endif
+
+            // Renormalise the variances if desired
+            if (renorm) {
+                // Statistics for renormalisation
+                psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe,
+                                                                                        "RENORM.MEAN"));
+                psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe,
+                                                                                         "RENORM.STDEV"));
+
+                if (!pmReadoutWeightRenormPixels(readout, maskBad, renormMean, renormStdev, rng)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
+                    psFree(output);
+                    return false;
+                }
+            }
 
             if (threads > 0) {
@@ -228,17 +369,40 @@
 
             // Do the image matching
-            if (!pmSubtractionMatch(output, NULL, readout, fake, footprint, regionSize, spacing, threshold,
-                                    sources, stampsName, type, size, order, widths, orders, inner, ringsOrder,
-                                    binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej,
-                                    sysError, maskVal, maskBad, maskPoor, poorFrac, badFrac,
-                                    PM_SUBTRACTION_MODE_1)) {
+            if (!pmSubtractionMatch(output, NULL, readout, fake, footprint, stride, regionSize, spacing,
+                                    threshold, stampSources, stampsName, type, size, order, widths, orders,
+                                    inner, ringsOrder, binning, penalty, optimum, optWidths, optOrder,
+                                    optThresh, iter, rej, sysError, maskVal, maskBad, maskPoor, poorFrac,
+                                    badFrac, PM_SUBTRACTION_MODE_1)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
                 psFree(fake);
                 psFree(optWidths);
+                psFree(stampSources);
                 psFree(output);
                 return false;
             }
+
+#ifdef TESTING
+            {
+                psString name = NULL;
+                psStringAppend(&name, "conv_%03d.fits", numInput);
+                psFits *fits = psFitsOpen(name, "w");
+                psFree(name);
+                psFitsWriteImage(fits, NULL, output->image, 0, NULL);
+                psFitsClose(fits);
+            }
+            {
+                psString name = NULL;
+                psStringAppend(&name, "diff_%03d.fits", numInput);
+                psFits *fits = psFitsOpen(name, "w");
+                psFree(name);
+                psBinaryOp(fake->image, output->image, "-", fake->image);
+                psFitsWriteImage(fits, NULL, fake->image, 0, NULL);
+                psFitsClose(fits);
+            }
+#endif
+
             psFree(fake);
             psFree(optWidths);
+            psFree(stampSources);
 
             if (threads > 0) {
@@ -283,4 +447,5 @@
         }
 
+#ifdef TESTING
         // Write convolution kernel
         {
@@ -297,5 +462,4 @@
             psFitsClose(fits);
         }
-#ifdef TESTING
     }
 #endif
@@ -355,7 +519,5 @@
         }
         psFree(iter);
-
-        float vf = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.VARFACTOR"); // Variance factor
-        *chi2 /= vf * num;
+        *chi2 /= num;
     }
 
@@ -383,6 +545,10 @@
         psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)",
                  psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV));
-        (void)psBinaryOp(readout->image, readout->image, "+",
-                         psScalarAlloc(- psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32));
+        (void)psBinaryOp(readout->image, readout->image, "-",
+                         psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32));
+        *weighting = 1.0 / PS_SQR(psStatsGetValue(bg, PS_STAT_ROBUST_STDEV));
+        psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "PPSTACK.WEIGHTING", 0,
+                         "Weighting by 1/noise^2 for stack",
+                         1.0 / PS_SQR(psStatsGetValue(bg, PS_STAT_ROBUST_STDEV)));
     }
     psFree(bg);
Index: branches/bills_081204/ppStack/src/ppStackPhotometry.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackPhotometry.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackPhotometry.c	(revision 20890)
@@ -100,6 +100,8 @@
         } else {
             source->psfMag = - 2.5 * log10(sumImage / sumKernel * M_PI * PS_SQR(zpRadius));
-            numGood++;
-            maxMag = PS_MAX(maxMag, source->psfMag);
+            if (isfinite(source->psfMag)) {
+                numGood++;
+                maxMag = PS_MAX(maxMag, source->psfMag);
+            }
         }
     }
@@ -143,4 +145,11 @@
     }
 
+    if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
+        !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") ||
+        !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files.");
+        return false;
+    }
+
     pmFPAfileActivate(config->files, false, "PSPHOT.INPUT");
 
Index: branches/bills_081204/ppStack/src/ppStackReadout.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackReadout.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackReadout.c	(revision 20890)
@@ -22,8 +22,9 @@
     psArray *subRegions = args->data[3]; // Regions for PSF-matching
     psArray *subKernels = args->data[4]; // Kernels for PSF-matching
-    psVector *addVariance = args->data[5]; // Additional variance when rejecting
-
-    psArray *inspect = ppStackReadoutInitial(config, outRO, thread->readouts,
-                                             subRegions, subKernels, addVariance);
+    psVector *weightings = args->data[5]; // Weightings (1/noise^2) for each image
+    psVector *addVariance = args->data[6]; // Additional variance when rejecting
+
+    psArray *inspect = ppStackReadoutInitial(config, outRO, thread->readouts, subRegions, subKernels,
+                                             weightings, addVariance);
 
     job->results = inspect;
@@ -42,6 +43,8 @@
     pmReadout *outRO = args->data[2];   // Output readout
     psArray *rejected = args->data[3];  // Rejected pixels
-
-    bool status = ppStackReadoutFinal(config, outRO, thread->readouts, rejected); // Status of operation
+    psVector *weightings = args->data[4];  // Weighting (1/noise^2) for each image
+
+    bool status = ppStackReadoutFinal(config, outRO, thread->readouts, rejected,
+                                      weightings); // Status of operation
 
     thread->busy = false;
@@ -69,4 +72,9 @@
     }
 
+    if (!output) {
+        // If there are no pixels to inspect, then just fake it
+        output = psPixelsAllocEmpty(0);
+    }
+
     psFree(inputs);
     inspect->data[index] = output;
@@ -77,5 +85,6 @@
 
 psArray *ppStackReadoutInitial(const pmConfig *config, pmReadout *outRO, const psArray *readouts,
-                               const psArray *regions, const psArray *kernels, const psVector *addVariance)
+                               const psArray *regions, const psArray *kernels, const psVector *weightings,
+                               const psVector *addVariance)
 {
     assert(config);
@@ -86,4 +95,5 @@
     assert(readouts->n == regions->n);
     assert(regions->n == kernels->n);
+    assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32);
     assert(addVariance && addVariance->n == readouts->n && addVariance->type.type == PS_TYPE_F32);
     static int sectionNum = 0;          // Section number; for debugging outputs
@@ -117,11 +127,15 @@
             continue;
         }
-        pmFPA *fpa = ro->parent->parent->parent; // Parent FPA
-
-        float weighting = psMetadataLookupF32(&mdok, fpa->analysis, "PPSTACK.WEIGHTING"); // Relative weight
+
+#if 0
+        // This doesn't seem to work, so getting the weightings directly from a vector
+        float weighting = psMetadataLookupF32(&mdok, ro->analysis, "PPSTACK.WEIGHTING"); // Relative weight
         if (!mdok || !isfinite(weighting)) {
-            psWarning("No WEIGHTING supplied for image %d --- set to unity.", i);
+            psWarning("No weighting supplied for image %d --- set to unity.", i);
             weighting = 1.0;
-        }
+        } else {
+            psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %f", i, weighting);
+        }
+#endif
 
         // Ensure there is a mask, or pmStackCombine will complain
@@ -131,5 +145,5 @@
         }
 
-        stack->data[i] = pmStackDataAlloc(ro, weighting, addVariance->data.F32[i]);
+        stack->data[i] = pmStackDataAlloc(ro, weightings->data.F32[i], addVariance->data.F32[i]);
     }
 
@@ -175,5 +189,5 @@
 
 bool ppStackReadoutFinal(const pmConfig *config, pmReadout *outRO, const psArray *readouts,
-                         const psArray *rejected)
+                         const psArray *rejected, const psVector *weightings)
 {
     assert(config);
@@ -182,4 +196,5 @@
     assert(rejected);
     assert(readouts->n == rejected->n);
+    assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32);
 
     static int sectionNum = 0;
@@ -210,12 +225,14 @@
         pmReadout *ro = readouts->data[i];
         assert(ro);
-        pmFPA *fpa = ro->parent->parent->parent; // Parent FPA
-
+
+#if 0
+        // This doesn't seem to work, so getting the weightings directly from a vector
         bool mdok;                      // Status of MD lookup
-        float weighting = psMetadataLookupF32(&mdok, fpa->analysis, "PPSTACK.WEIGHTING"); // Relative weight
+        float weighting = psMetadataLookupF32(&mdok, ro->analysis, "PPSTACK.WEIGHTING"); // Relative weight
         if (!mdok || !isfinite(weighting)) {
             psWarning("No WEIGHTING supplied for image %d --- set to unity.", i);
             weighting = 1.0;
         }
+#endif
 
         // Ensure there is a mask, or pmStackCombine will complain
@@ -225,5 +242,5 @@
         }
 
-        pmStackData *data = pmStackDataAlloc(ro, weighting, NAN);
+        pmStackData *data = pmStackDataAlloc(ro, weightings->data.F32[i], NAN);
         data->reject = psMemIncrRefCounter(rejected->data[i]);
         stack->data[i] = data;
Index: branches/bills_081204/ppStack/src/ppStackThread.c
===================================================================
--- branches/cnb_branch_20081104/ppStack/src/ppStackThread.c	(revision 20530)
+++ branches/bills_081204/ppStack/src/ppStackThread.c	(revision 20890)
@@ -235,5 +235,5 @@
 
     {
-        psThreadTask *task = psThreadTaskAlloc("PPSTACK_INITIAL_COMBINE", 6);
+        psThreadTask *task = psThreadTaskAlloc("PPSTACK_INITIAL_COMBINE", 7);
         task->function = &ppStackReadoutInitialThread;
         psThreadTaskAdd(task);
@@ -249,5 +249,5 @@
 
     {
-        psThreadTask *task = psThreadTaskAlloc("PPSTACK_FINAL_COMBINE", 4);
+        psThreadTask *task = psThreadTaskAlloc("PPSTACK_FINAL_COMBINE", 5);
         task->function = &ppStackReadoutFinalThread;
         psThreadTaskAdd(task);
Index: branches/bills_081204/ppStats/src/ppStatsFromMetadataStats.c
===================================================================
--- branches/cnb_branch_20081104/ppStats/src/ppStatsFromMetadataStats.c	(revision 20530)
+++ branches/bills_081204/ppStats/src/ppStatsFromMetadataStats.c	(revision 20890)
@@ -42,5 +42,5 @@
         double value;
         if (!strcasecmp (entry->statistic, "RMS")) {
-            value = sqrt(stats->sampleMean * entry->vector->n);
+            value = sqrt(stats->sampleMean);
             goto got_value;
         }
Index: branches/bills_081204/ppStats/src/ppStatsSetupFromArgs.c
===================================================================
--- branches/cnb_branch_20081104/ppStats/src/ppStatsSetupFromArgs.c	(revision 20530)
+++ branches/bills_081204/ppStats/src/ppStatsSetupFromArgs.c	(revision 20890)
@@ -160,5 +160,5 @@
 
         psMetadata *header = psFitsReadHeader(NULL, data->fits); // The FITS (primary) header
-        psMetadata *format = pmConfigCameraFormatFromHeader(NULL, NULL, config, header, true);
+        psMetadata *format = pmConfigCameraFormatFromHeader(NULL, NULL, NULL, config, header, true);
         if (!format) {
             psError(PS_ERR_UNKNOWN, false, "Unable to determine camera format for %s\n", inName);
Index: branches/bills_081204/ppSub/src/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/ppSub/src/Makefile.am	(revision 20530)
+++ branches/bills_081204/ppSub/src/Makefile.am	(revision 20890)
@@ -7,8 +7,9 @@
 	ppSub.c			\
 	ppSubArguments.c	\
+	ppSubBackground.c	\
 	ppSubCamera.c		\
 	ppSubLoop.c		\
 	ppSubReadout.c		\
-	ppSubVersion.c            
+	ppSubVersion.c
 
 ppSubKernel_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSUB_CFLAGS)
Index: branches/bills_081204/ppSub/src/ppSub.h
===================================================================
--- branches/cnb_branch_20081104/ppSub/src/ppSub.h	(revision 20530)
+++ branches/bills_081204/ppSub/src/ppSub.h	(revision 20890)
@@ -27,4 +27,10 @@
     );
 
+/// Higher-order background subtraction
+bool ppSubBackground(pmReadout *ro,     ///< Readout of interest
+                     pmConfig *config,  ///< Configuration
+                     const pmFPAview *view ///< View to readout
+    );
+
 /// Put the program version information into a metadata
 void ppSubVersionMetadata(psMetadata *metadata ///< Metadata to populate
Index: branches/bills_081204/ppSub/src/ppSubArguments.c
===================================================================
--- branches/cnb_branch_20081104/ppSub/src/ppSubArguments.c	(revision 20530)
+++ branches/bills_081204/ppSub/src/ppSubArguments.c	(revision 20890)
@@ -211,4 +211,5 @@
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-spacing", 0, "Typical stamp spacing (pixels)", NAN);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-footprint", 0, "Stamp footprint half-size (pixels)", -1);
+    psMetadataAddS32(arguments, PS_LIST_TAIL, "-stride", 0, "Size of convolution patches (pixels)", -1);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold", 0, "Minimum threshold for stamps (ADU)", NAN);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-iter", 0, "Number of rejection iterations", -1);
@@ -300,4 +301,5 @@
     VALUE_ARG_RECIPE_INT("-spam-binning", "SPAM.BINNING",    S32, 0);
     VALUE_ARG_RECIPE_INT("-footprint",    "STAMP.FOOTPRINT", S32, -1);
+    VALUE_ARG_RECIPE_INT("-stride",       "STRIDE",          S32, -1);
     VALUE_ARG_RECIPE_FLOAT("-threshold",  "STAMP.THRESHOLD", F32);
     VALUE_ARG_RECIPE_INT("-iter",         "ITER",            S32, -1);
Index: branches/bills_081204/ppSub/src/ppSubBackground.c
===================================================================
--- branches/bills_081204/ppSub/src/ppSubBackground.c	(revision 20890)
+++ branches/bills_081204/ppSub/src/ppSubBackground.c	(revision 20890)
@@ -0,0 +1,79 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+#include <psphot.h>
+#include "ppSub.h"
+
+// Copied from ppImageSubtractBackground()
+bool ppSubBackground(pmReadout *ro, pmConfig *config, const pmFPAview *view)
+{
+    psAssert(config, "Need configuration");
+    psAssert(view, "Need view to chip");
+
+    // The background model file may be defined, though the model hasn't been built
+    // If so, we will build it below.
+    bool status;                        // Status of lookup
+    pmFPAfile *modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model
+    if (!status) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "PSPHOT.BACKMDL file is not defined");
+        return false;
+    }
+
+    psMetadata *ppSubRecipe = psMetadataLookupPtr(NULL, config->recipes, PPSUB_RECIPE);
+    psAssert(ppSubRecipe, "Need PPSUB recipe");
+    psMetadata *psphotRecipe = psMetadataLookupPtr(NULL, config->recipes, PSPHOT_RECIPE);
+    psAssert(psphotRecipe, "Need PSPHOT recipe");
+
+    psString maskBadStr = psMetadataLookupStr(NULL, ppSubRecipe, "MASK.BAD"); // Name of bits to mask for bad
+    psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
+
+    // user-defined masks to test for good/bad pixels (build from recipe list if not yet set)
+    psMetadataAddU8(psphotRecipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", maskBad);
+
+    psImage *image = ro->image, *mask = ro->mask; // Image and mask of interest
+
+    pmReadout *modelRO = pmFPAviewThisReadout(view, modelFile->fpa); // Background model
+    if (!modelRO) {
+        // Create the background model
+        if (!psphotModelBackground(config, view, "PPSUB.OUTPUT")) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to model background");
+            return false;
+        }
+        modelRO = (modelFile->mode == PM_FPA_MODE_INTERNAL) ? modelFile->readout :
+                   pmFPAviewThisReadout(view, modelFile->fpa);
+        if (!modelRO) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background model");
+            return false;
+        }
+    }
+    psImageBinning *binning = psMetadataLookupPtr(&status, psphotRecipe,
+                                                  "PSPHOT.BACKGROUND.BINNING"); // Binning for model
+    psImage *modelImage = modelRO->image; // Background model
+
+    // Do the background subtraction
+    int numCols = image->numCols, numRows = image->numRows; // Size of image
+    for (int y = 0; y < numRows; y++) {
+        for (int x = 0; x < numCols; x++) {
+            if (mask && mask->data.PS_TYPE_MASK_DATA[y][x] & maskBad) {
+                image->data.F32[y][x] = 0.0;
+            } else {
+                float value = psImageUnbinPixel(x + 0.5, y + 0.5, modelImage, binning); // Background value
+                if (!isfinite(value)) {
+                    image->data.F32[y][x] = NAN;
+                    mask->data.PS_TYPE_MASK_DATA[y][x] |= maskBad;
+                } else {
+                    image->data.F32[y][x] -= value;
+                }
+            }
+        }
+    }
+
+    pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL");
+    pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL.STDEV");
+
+    return true;
+}
Index: branches/bills_081204/ppSub/src/ppSubCamera.c
===================================================================
--- branches/cnb_branch_20081104/ppSub/src/ppSubCamera.c	(revision 20530)
+++ branches/bills_081204/ppSub/src/ppSubCamera.c	(revision 20890)
@@ -179,18 +179,5 @@
     if (psMetadataLookupBool(NULL, recipe, "PHOTOMETRY")) {
         psphotModelClassInit ();        // load implementation-specific models
-        pmFPAfile *psphot = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT");
-        if (!psphot) {
-            psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.INPUT");
-            return false;
-        }
-        if (psphot->type != PM_FPA_FILE_IMAGE) {
-            psError(PS_ERR_IO, true, "PSPHOT.INPUT is not of type IMAGE");
-            return false;
-        }
-
-        if (!psphotDefineFiles(config, psphot)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to set up psphot files.");
-            return false;
-        }
+
 
         // Internal-ish file for getting the PSF from the matched addition
@@ -205,4 +192,22 @@
         }
         pmFPAfileActivate(config->files, false, "PSPHOT.PSF.LOAD");
+    }
+
+    // Always do this, since we're using psphot for the background model.
+    {
+        pmFPAfile *psphot = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT");
+        if (!psphot) {
+            psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.INPUT");
+            return false;
+        }
+        if (psphot->type != PM_FPA_FILE_IMAGE) {
+            psError(PS_ERR_IO, true, "PSPHOT.INPUT is not of type IMAGE");
+            return false;
+        }
+
+        if (!psphotDefineFiles(config, psphot)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to set up psphot files.");
+            return false;
+        }
 
     }
Index: branches/bills_081204/ppSub/src/ppSubReadout.c
===================================================================
--- branches/cnb_branch_20081104/ppSub/src/ppSubReadout.c	(revision 20530)
+++ branches/bills_081204/ppSub/src/ppSubReadout.c	(revision 20890)
@@ -91,4 +91,5 @@
     int footprint = psMetadataLookupS32(NULL, recipe, "STAMP.FOOTPRINT"); // Stamp half-size
     float threshold = psMetadataLookupF32(NULL, recipe, "STAMP.THRESHOLD"); // Threshold for stmps
+    int stride = psMetadataLookupS32(NULL, recipe, "STRIDE"); // Size of convolution patches
     int iter = psMetadataLookupS32(NULL, recipe, "ITER"); // Rejection iterations
     float rej = psMetadataLookupF32(NULL, recipe, "REJ"); // Rejection threshold
@@ -118,4 +119,15 @@
     bool dual = psMetadataLookupBool(&mdok, recipe, "DUAL"); // Dual convolution?
 
+    // Statistics for renormalisation
+    psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "RENORM.MEAN"));
+    psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "RENORM.STDEV"));
+    if (renormMean == PS_STAT_NONE || renormStdev == PS_STAT_NONE) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
+                "Unable to parse renormalisation statistics from recipe.");
+        return false;
+    }
+    int renormNum = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); // Number of samples for renormalisation
+    float renormWidth = psMetadataLookupS32(&mdok, recipe, "RENORM.WIDTH"); // Width of Gaussian phot
+
     psString interpModeStr = psMetadataLookupStr(&mdok, recipe, "INTERPOLATION"); // Interpolation mode
     psImageInterpolateMode interpMode = psImageInterpolateModeFromString(interpModeStr); // Interp
@@ -180,8 +192,8 @@
 
     // Match the PSFs
-    if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, regionSize, spacing, threshold, sources,
-                            stampsName, type, size, order, widths, orders, inner, ringsOrder, binning,
-                            penalty, optimum, optWidths, optOrder, optThresh, iter, rej, sys, maskVal,
-                            maskBad, maskPoor, poorFrac, badFrac, subMode)) {
+    if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing, threshold,
+                            sources, stampsName, type, size, order, widths, orders, inner, ringsOrder,
+                            binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej, sys,
+                            maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
         psFree(inConv);
@@ -356,17 +368,4 @@
 
         if (psMetadataLookupBool(&mdok, recipe, "RENORM")) {
-            // Statistics for renormalisation
-            psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe,
-                                                                                    "RENORM.MEAN"));
-            psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe,
-                                                                                     "RENORM.STDEV"));
-            if (renormMean == PS_STAT_NONE || renormStdev == PS_STAT_NONE) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                        "Unable to parse renormalisation statistics from recipe.");
-                psFree(outRO);
-                return false;
-            }
-            int renormNum = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); // Number of samples
-            float renormWidth = psMetadataLookupS32(&mdok, recipe, "RENORM.WIDTH"); // Width of Gaussian phot
             psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
             if (!pmReadoutWeightRenormPhot(outRO, maskValue, renormNum, renormWidth,
@@ -412,4 +411,11 @@
         }
 
+        if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
+            !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") ||
+            !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files.");
+            return false;
+        }
+
         if (breakpoint) {
             psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE,
@@ -439,10 +445,12 @@
     outRO->mask = (psImage*)psBinaryOp(outRO->mask, inConv->mask, "|", refConv->mask);
 
-    outRO->data_exists = outCell->data_exists = outCell->parent->data_exists = true;
-
-    pmReadoutMaskApply(outRO, maskBad);
-
     psFree(inConv);
     psFree(refConv);
+
+    outRO->data_exists = outCell->data_exists = outCell->parent->data_exists = true;
+
+#if 0
+    pmReadoutMaskApply(outRO, maskBad);
+#endif
 
 #ifdef TESTING
@@ -460,27 +468,4 @@
         psFree(outRO);
         return false;
-    }
-
-    if (psMetadataLookupBool(&mdok, recipe, "RENORM")) {
-        // Statistics for renormalisation
-        psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe,
-                                                                                "RENORM.MEAN"));
-        psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe,
-                                                                                 "RENORM.STDEV"));
-        if (renormMean == PS_STAT_NONE || renormStdev == PS_STAT_NONE) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                    "Unable to parse renormalisation statistics from recipe.");
-            psFree(outRO);
-            return false;
-        }
-        int renormNum = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); // Number of samples
-        float renormWidth = psMetadataLookupS32(&mdok, recipe, "RENORM.WIDTH"); // Width of Gaussian phot
-        psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
-        if (!pmReadoutWeightRenormPhot(outRO, maskValue, renormNum, renormWidth,
-                                       renormMean, renormStdev, NULL)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
-            psFree(outRO);
-            return false;
-        }
     }
 
@@ -562,4 +547,14 @@
         }
 
+        if (psMetadataLookupBool(&mdok, recipe, "RENORM")) {
+            psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
+            if (!pmReadoutWeightRenormPhot(outRO, maskValue, renormNum, renormWidth,
+                                           renormMean, renormStdev, NULL)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
+                psFree(outRO);
+                return false;
+            }
+        }
+
         // Need to ensure aperture residual is not calculated
         psMetadata *psphotRecipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe
@@ -578,4 +573,11 @@
         }
 
+        if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
+            !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") ||
+            !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files.");
+            return false;
+        }
+
         pmFPAfileActivate(config->files, false, "PSPHOT.INPUT");
         pmFPAfileActivate(config->files, false, "PSPHOT.LOAD.PSF");
@@ -584,5 +586,6 @@
             pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with the sources
             psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
-            psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", sources->n);
+            psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected",
+                             sources ? sources->n : 0);
             psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry",
                              psTimerClear("PPSUB_PHOT"));
@@ -595,5 +598,6 @@
             psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
             FILE *sourceFile = fopen("sources.dat", "w"); // File for sources
-            fprintf(sourceFile, "# x y mag mag_err psf_chisq cr_nsigma ext_nsigma psf_qf flags m_x m_y m_xx m_xy m_yy\n");
+            fprintf(sourceFile,
+                    "# x y mag mag_err psf_chisq cr_nsigma ext_nsigma psf_qf flags m_x m_y m_xx m_xy m_yy\n");
             for (int i = 0; i < sources->n; i++) {
                 pmSource *source = sources->data[i];
@@ -636,4 +640,21 @@
     }
 
+    // Higher order background subtraction using psphot
+    if (!ppSubBackground(outRO, config, view)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to subtract background.");
+        psFree(outRO);
+        return false;
+    }
+
+    // Renormalising for pixels, because that's what magic desires
+    if (psMetadataLookupBool(&mdok, recipe, "RENORM")) {
+        psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
+        if (!pmReadoutWeightRenormPixels(outRO, maskValue, renormMean, renormStdev, NULL)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
+            psFree(outRO);
+            return false;
+        }
+    }
+
     // Copy astrometry over
     pmFPA *refFPA = refRO->parent->parent->parent; // Reference FPA
Index: branches/bills_081204/psLib/src/astro/psTime.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/astro/psTime.c	(revision 20530)
+++ branches/bills_081204/psLib/src/astro/psTime.c	(revision 20890)
@@ -10,6 +10,6 @@
  *  @author Ross Harman, MHPCC
  *
- *  @version $Revision: 1.116 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-09-24 20:04:32 $
+ *  @version $Revision: 1.117 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-09 00:30:07 $
  *
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -1391,4 +1391,9 @@
 
     // Check valid year range
+    if ((time->sec) < ((psS64)YEAR_0000_SEC) || (time->sec) > ((psS64)YEAR_9999_SEC)) {
+      psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Error: %s, %" PRId64 ", is out of range.  Must be between %" PRId64 " and %" PRId64 ".", "time->sec", time->sec, ((psS64)YEAR_0000_SEC), ((psS64)YEAR_9999_SEC));
+      return NULL;
+    }
+
     PS_ASSERT_S64_WITHIN_RANGE(time->sec, (psS64)YEAR_0000_SEC, (psS64)YEAR_9999_SEC, NULL);
 
Index: branches/bills_081204/psLib/src/fft/psImageFFT.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/fft/psImageFFT.c	(revision 20530)
+++ branches/bills_081204/psLib/src/fft/psImageFFT.c	(revision 20890)
@@ -6,6 +6,6 @@
 /// @author Robert DeSonia, MHPCC
 ///
-/// @version $Revision: 1.27 $ $Name: not supported by cvs2svn $
-/// @date $Date: 2008-08-14 03:23:13 $
+/// @version $Revision: 1.28 $ $Name: not supported by cvs2svn $
+/// @date $Date: 2008-11-06 23:36:20 $
 ///
 /// Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -20,4 +20,5 @@
 #include <complex.h>
 #include <fftw3.h>
+#include <limits.h>
 #include <pthread.h>
 
@@ -32,4 +33,6 @@
 #include "psFFT.h"
 #include "psImageFFT.h"
+
+#define FFT_CONVOLVE_BINARY_SIZE 1               // Scale up to next binary size
 
 // Lock FFTW access
@@ -340,6 +343,24 @@
         return NULL;
     }
+
     int paddedCols = numCols + PS_MAX(-xMin, xMax); // Number of columns in padded image
     int paddedRows = numRows + PS_MAX(-yMin, yMax); // Number of rows in padded image
+
+#if CONVOLVE_FFT_BINARY_SIZE
+    // Make the size an integer power of two
+    {
+        int twoCols, twoRows;           // Size that is a factor of two
+        for (twoCols = 1; twoCols <= paddedCols && twoCols < INT_MAX - 1; twoCols <<= 1); // No action
+        for (twoRows = 1; twoRows <= paddedRows && twoRows < INT_MAX - 1; twoRows <<= 1); // No action
+        if (paddedCols > twoCols || paddedRows > twoRows) {
+            psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Unable to scale size (%dx%d) up to factor of two",
+                    paddedCols, paddedRows);
+            return NULL;
+        }
+        paddedCols = twoCols;
+        paddedRows = twoRows;
+    }
+#endif
+
     int numPadded = paddedCols * paddedRows; // Number of pixels in padded image
 
Index: branches/bills_081204/psLib/src/fits/psFitsHeader.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/fits/psFitsHeader.c	(revision 20530)
+++ branches/bills_081204/psLib/src/fits/psFitsHeader.c	(revision 20890)
@@ -7,6 +7,6 @@
  *  @author Robert DeSonia, MHPCC
  *
- *  @version $Revision: 1.48 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-03 01:43:43 $
+ *  @version $Revision: 1.49 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-10 23:31:12 $
  *
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -54,5 +54,5 @@
 // List of compressed FITS header keys not to write (handled by cfitsio); NULL-terminated
 static const char *noWriteCompressedKeys[] = { "ZBITPIX", "ZIMAGE", "ZBITPIX", "ZCMPTYPE", "ZSIMPLE",
-                                               "ZEXTEND", "ZBLANK", NULL };
+                                               "ZEXTEND", "ZBLANK", "ZDATASUM", "ZHECKSUM", NULL };
 
 // List of the start of FITS header keys not to write (handled by cfitsio); NULL-terminated
Index: branches/bills_081204/psLib/src/imageops/psImageBinning.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageBinning.c	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageBinning.c	(revision 20890)
@@ -8,6 +8,6 @@
  *  @author Eugene Magnier, IfA
  *
- *  @version $Revision: 1.7 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2007-11-08 03:10:30 $
+ *  @version $Revision: 1.8 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-17 02:38:07 $
  *
  *  Copyright 2007 Institute for Astronomy, University of Hawaii
@@ -35,4 +35,14 @@
     psMemSetDeallocator(binning, (psFreeFunc)psImageBinningFree);
 
+    binning->nXfine = 0;
+    binning->nYfine = 0;
+    binning->nXruff = 0;
+    binning->nYruff = 0;
+    binning->nXbin = 0;
+    binning->nYbin = 0;
+    binning->nXoff = 0;
+    binning->nYoff = 0;
+    binning->nXskip = 0;
+    binning->nYskip = 0;
     return binning;
 }
@@ -143,8 +153,8 @@
     psRegion fineRegion;
 
-    fineRegion.x0 = ruffRegion.x0 * binning->nXbin;
-    fineRegion.x1 = ruffRegion.x1 * binning->nXbin;
-    fineRegion.y0 = ruffRegion.y0 * binning->nYbin;
-    fineRegion.y1 = ruffRegion.y1 * binning->nYbin;
+    fineRegion.x0 = ruffRegion.x0 * binning->nXbin + binning->nXskip;
+    fineRegion.x1 = ruffRegion.x1 * binning->nXbin + binning->nXskip;
+    fineRegion.y0 = ruffRegion.y0 * binning->nYbin + binning->nYskip;
+    fineRegion.y1 = ruffRegion.y1 * binning->nYbin + binning->nYskip;
     return fineRegion;
 }
@@ -154,8 +164,8 @@
     psRegion ruffRegion;
 
-    ruffRegion.x0 = fineRegion.x0 / binning->nXbin;
-    ruffRegion.x1 = fineRegion.x1 / binning->nXbin;
-    ruffRegion.y0 = fineRegion.y0 / binning->nYbin;
-    ruffRegion.y1 = fineRegion.y1 / binning->nYbin;
+    ruffRegion.x0 = (fineRegion.x0 - binning->nXskip) / binning->nXbin;
+    ruffRegion.x1 = (fineRegion.x1 - binning->nXskip) / binning->nXbin;
+    ruffRegion.y0 = (fineRegion.y0 - binning->nYskip) / binning->nYbin;
+    ruffRegion.y1 = (fineRegion.y1 - binning->nYskip) / binning->nYbin;
     return ruffRegion;
 }
Index: branches/bills_081204/psLib/src/imageops/psImageConvolve.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageConvolve.c	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageConvolve.c	(revision 20890)
@@ -7,6 +7,6 @@
 /// @author Eugene Magnier, IfA
 ///
-/// @version $Revision: 1.80 $ $Name: not supported by cvs2svn $
-/// @date $Date: 2008-10-29 20:43:18 $
+/// @version $Revision: 1.81 $ $Name: not supported by cvs2svn $
+/// @date $Date: 2008-11-26 00:43:12 $
 ///
 /// Copyright 2004-2007 Institute for Astronomy, University of Hawaii
@@ -750,76 +750,76 @@
     switch (image->type.type) {
       case PS_TYPE_F32: {
-	  psImage *calculation = psImageAlloc(numRows, numCols, PS_TYPE_F32); /* Calculation image; BW */
-	  psImage *calcMask = psImageAlloc(numRows, numCols, PS_TYPE_MASK); /* Mask for calculation image; BW */
-   
-	  /** Smooth in X direction **/
-	  for (int j = 0; j < numRows; j++) {
-	      for (int i = 0; i < numCols; i++) {
-		  int xMin = PS_MAX(i - size, 0);
-		  int xMax = PS_MIN(i + size, xLast);
-		  const psMaskType *maskData = &mask->data.PS_TYPE_MASK_DATA[j][xMin];
-		  const psF32 *imageData = &image->data.F32[j][xMin];
-		  int uMin = - PS_MIN(i, size); /* Minimum kernel index */
-		  const psF32 *gaussData = &gauss[uMin];
-		  double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
-		  for (int x = xMin; x <= xMax; x++, maskData++, imageData++, gaussData++) {
-		      if (*maskData & maskVal) {
-			  continue;
-		      }
-		      // if ((i == 1000) && (j > 10) && (j < 15)) {
-		      // 	  fprintf (stderr, "%d %d  %d  %f  %f   %f %f\n", i, j, x, *imageData, *gaussData, sumIG, sumG);
-		      // }
-		      sumIG += *imageData * *gaussData;
-		      sumG += *gaussData;
-		  }
-		  if (sumG > minGauss) {
-		      /* BW */
-		      calculation->data.F32[i][j] = sumIG / sumG;
-		      calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0;
-		  } else {
-		      /* BW */
-		      calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0xFF;
-		  }
-	      }
-	  }
-   
-	  output = psImageRecycle(output, numCols, numRows, PS_TYPE_F32);
-   
-	  /** Smooth in Y direction  **/
-	  for (int i = 0; i < numCols; i++) {
-	      for (int j = 0; j < numRows; j++) {
-		  int yMin = PS_MAX(j - size, 0);
-		  int yMax = PS_MIN(j + size, yLast);
-		  const psMaskType *maskData = &calcMask->data.PS_TYPE_MASK_DATA[i][yMin]; /* BW */
-		  const psF32 *imageData = &calculation->data.F32[i][yMin]; /* BW */
-		  int vMin = - PS_MIN(j, size); /* Minimum kernel index */
-		  const psF32 *gaussData = &gauss[vMin];
-		  double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
-		  for (int y = yMin; y <= yMax; y++, maskData++, imageData++, gaussData++) {
-		      if (*maskData) {
-			  continue;
-		      }
-		      sumIG += *imageData * *gaussData;
-		      sumG += *gaussData;
-		  }
-		  output->data.F32[j][i] = (sumG > minGauss) ? sumIG / sumG : NAN;
-	      }
-	  }
-   
-	  psFree(calculation);
-	  psFree(calcMask);
-	  psFree(gaussNorm);
-   
-	  return output;
+          psImage *calculation = psImageAlloc(numRows, numCols, PS_TYPE_F32); /* Calculation image; BW */
+          psImage *calcMask = psImageAlloc(numRows, numCols, PS_TYPE_MASK); /* Mask for calculation image; BW */
+
+          /** Smooth in X direction **/
+          for (int j = 0; j < numRows; j++) {
+              for (int i = 0; i < numCols; i++) {
+                  int xMin = PS_MAX(i - size, 0);
+                  int xMax = PS_MIN(i + size, xLast);
+                  const psMaskType *maskData = &mask->data.PS_TYPE_MASK_DATA[j][xMin];
+                  const psF32 *imageData = &image->data.F32[j][xMin];
+                  int uMin = - PS_MIN(i, size); /* Minimum kernel index */
+                  const psF32 *gaussData = &gauss[uMin];
+                  double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
+                  for (int x = xMin; x <= xMax; x++, maskData++, imageData++, gaussData++) {
+                      if (*maskData & maskVal) {
+                          continue;
+                      }
+                      // if ((i == 1000) && (j > 10) && (j < 15)) {
+                      //          fprintf (stderr, "%d %d  %d  %f  %f   %f %f\n", i, j, x, *imageData, *gaussData, sumIG, sumG);
+                      // }
+                      sumIG += *imageData * *gaussData;
+                      sumG += *gaussData;
+                  }
+                  if (sumG > minGauss) {
+                      /* BW */
+                      calculation->data.F32[i][j] = sumIG / sumG;
+                      calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0;
+                  } else {
+                      /* BW */
+                      calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0xFF;
+                  }
+              }
+          }
+
+          output = psImageRecycle(output, numCols, numRows, PS_TYPE_F32);
+
+          /** Smooth in Y direction  **/
+          for (int i = 0; i < numCols; i++) {
+              for (int j = 0; j < numRows; j++) {
+                  int yMin = PS_MAX(j - size, 0);
+                  int yMax = PS_MIN(j + size, yLast);
+                  const psMaskType *maskData = &calcMask->data.PS_TYPE_MASK_DATA[i][yMin]; /* BW */
+                  const psF32 *imageData = &calculation->data.F32[i][yMin]; /* BW */
+                  int vMin = - PS_MIN(j, size); /* Minimum kernel index */
+                  const psF32 *gaussData = &gauss[vMin];
+                  double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
+                  for (int y = yMin; y <= yMax; y++, maskData++, imageData++, gaussData++) {
+                      if (*maskData) {
+                          continue;
+                      }
+                      sumIG += *imageData * *gaussData;
+                      sumG += *gaussData;
+                  }
+                  output->data.F32[j][i] = (sumG > minGauss) ? sumIG / sumG : NAN;
+              }
+          }
+
+          psFree(calculation);
+          psFree(calcMask);
+          psFree(gaussNorm);
+
+          return output;
       }
-	IMAGESMOOTH_MASK_CASE(F64);
+        IMAGESMOOTH_MASK_CASE(F64);
       default:
-	psFree(gaussNorm);
-	char *typeStr;
-	PS_TYPE_NAME(typeStr,image->type.type);
-	psError(PS_ERR_BAD_PARAMETER_TYPE, true,
-		_("Specified psImage type, %s, is not supported."),
-		typeStr);
-	return false;
+        psFree(gaussNorm);
+        char *typeStr;
+        PS_TYPE_NAME(typeStr,image->type.type);
+        psError(PS_ERR_BAD_PARAMETER_TYPE, true,
+                _("Specified psImage type, %s, is not supported."),
+                typeStr);
+        return false;
     }
 
@@ -828,14 +828,14 @@
 }
 
-bool psImageSmoothMask_ScanRows_F32 (psImage *calculation, 
-				     psImage *calcMask, 
-				     const psImage *image, 
-				     const psImage *mask, 
-				     psMaskType maskVal,
-				     psVector *gaussNorm, 
-				     float minGauss, 
-				     int size, 
-				     int rowStart, 
-				     int rowStop) 
+bool psImageSmoothMask_ScanRows_F32 (psImage *calculation,
+                                     psImage *calcMask,
+                                     const psImage *image,
+                                     const psImage *mask,
+                                     psMaskType maskVal,
+                                     psVector *gaussNorm,
+                                     float minGauss,
+                                     int size,
+                                     int rowStart,
+                                     int rowStop)
 {
     const psF32 *gauss = &gaussNorm->data.F32[size]; // Gaussian convolution kernel
@@ -845,28 +845,28 @@
 
     for (int j = rowStart; j < rowStop; j++) {
-	for (int i = 0; i < numCols; i++) {
-	    int xMin = PS_MAX(i - size, 0);
-	    int xMax = PS_MIN(i + size, xLast);
-	    const psMaskType *maskData = &mask->data.PS_TYPE_MASK_DATA[j][xMin];
-	    const psF32 *imageData = &image->data.F32[j][xMin];
-	    int uMin = - PS_MIN(i, size); /* Minimum kernel index */
-	    const psF32 *gaussData = &gauss[uMin];
-	    double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
-	    for (int x = xMin; x <= xMax; x++, maskData++, imageData++, gaussData++) {
-		if (*maskData & maskVal) {
-		    continue;
-		}
-		sumIG += *imageData * *gaussData;
-		sumG += *gaussData;
-	    }
-	    if (sumG > minGauss) {
-		/* BW */
-		calculation->data.F32[i][j] = sumIG / sumG;
-		calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0;
-	    } else {
-		/* BW */
-		calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0xFF;
-	    }
-	}
+        for (int i = 0; i < numCols; i++) {
+            int xMin = PS_MAX(i - size, 0);
+            int xMax = PS_MIN(i + size, xLast);
+            const psMaskType *maskData = &mask->data.PS_TYPE_MASK_DATA[j][xMin];
+            const psF32 *imageData = &image->data.F32[j][xMin];
+            int uMin = - PS_MIN(i, size); /* Minimum kernel index */
+            const psF32 *gaussData = &gauss[uMin];
+            double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
+            for (int x = xMin; x <= xMax; x++, maskData++, imageData++, gaussData++) {
+                if (*maskData & maskVal) {
+                    continue;
+                }
+                sumIG += *imageData * *gaussData;
+                sumG += *gaussData;
+            }
+            if (sumG > minGauss) {
+                /* BW */
+                calculation->data.F32[i][j] = sumIG / sumG;
+                calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0;
+            } else {
+                /* BW */
+                calcMask->data.PS_TYPE_MASK_DATA[i][j] = 0xFF;
+            }
+        }
     }
     return true;
@@ -874,12 +874,12 @@
 
 bool psImageSmoothMask_ScanCols_F32 (psImage *output,
-				     psImage *calculation, 
-				     psImage *calcMask, 
-				     psMaskType maskVal,
-				     psVector *gaussNorm, 
-				     float minGauss, 
-				     int size, 
-				     int colStart, 
-				     int colStop) 
+                                     psImage *calculation,
+                                     psImage *calcMask,
+                                     psMaskType maskVal,
+                                     psVector *gaussNorm,
+                                     float minGauss,
+                                     int size,
+                                     int colStart,
+                                     int colStop)
 {
     const psF32 *gauss = &gaussNorm->data.F32[size]; // Gaussian convolution kernel
@@ -889,25 +889,25 @@
 
     for (int i = colStart; i < colStop; i++) {
-	for (int j = 0; j < numRows; j++) {
-	    int yMin = PS_MAX(j - size, 0);
-	    int yMax = PS_MIN(j + size, yLast);
-	    const psMaskType *maskData = &calcMask->data.PS_TYPE_MASK_DATA[i][yMin]; /* BW */
-	    const psF32 *imageData = &calculation->data.F32[i][yMin]; /* BW */
-	    int vMin = - PS_MIN(j, size); /* Minimum kernel index */
-	    const psF32 *gaussData = &gauss[vMin];
-	    double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
-	    for (int y = yMin; y <= yMax; y++, maskData++, imageData++, gaussData++) {
-		if (*maskData) {
-		    continue;
-		}
-		sumIG += *imageData * *gaussData;
-		sumG += *gaussData;
-	    }
-	    output->data.F32[j][i] = (sumG > minGauss) ? sumIG / sumG : NAN;
-	}
+        for (int j = 0; j < numRows; j++) {
+            int yMin = PS_MAX(j - size, 0);
+            int yMax = PS_MIN(j + size, yLast);
+            const psMaskType *maskData = &calcMask->data.PS_TYPE_MASK_DATA[i][yMin]; /* BW */
+            const psF32 *imageData = &calculation->data.F32[i][yMin]; /* BW */
+            int vMin = - PS_MIN(j, size); /* Minimum kernel index */
+            const psF32 *gaussData = &gauss[vMin];
+            double sumIG = 0.0, sumG = 0.0; /* Sums for convolution */
+            for (int y = yMin; y <= yMax; y++, maskData++, imageData++, gaussData++) {
+                if (*maskData) {
+                    continue;
+                }
+                sumIG += *imageData * *gaussData;
+                sumG += *gaussData;
+            }
+            output->data.F32[j][i] = (sumG > minGauss) ? sumIG / sumG : NAN;
+        }
     }
     return true;
 }
-   
+
 bool psImageSmoothMask_ScanRows_F32_Threaded(psThreadJob *job)
 {
@@ -921,13 +921,13 @@
     const psImage *mask   = job->args->data[3]; // input mask
 
-    psMaskType maskVal 	  = PS_SCALAR_VALUE(job->args->data[4],U8);
-    psVector *gaussNorm	  = job->args->data[5]; // gauss kernel
-    float minGauss    	  = PS_SCALAR_VALUE(job->args->data[6],F32);
-    int size           	  = PS_SCALAR_VALUE(job->args->data[7],S32);
-    int rowStart       	  = PS_SCALAR_VALUE(job->args->data[8],S32);
-    int rowStop        	  = PS_SCALAR_VALUE(job->args->data[9],S32);
-    return psImageSmoothMask_ScanRows_F32(calculation, calcMask, image, mask, maskVal, 
-					  gaussNorm, minGauss, size, 
-					  rowStart, rowStop);
+    psMaskType maskVal    = PS_SCALAR_VALUE(job->args->data[4],U8);
+    psVector *gaussNorm   = job->args->data[5]; // gauss kernel
+    float minGauss        = PS_SCALAR_VALUE(job->args->data[6],F32);
+    int size              = PS_SCALAR_VALUE(job->args->data[7],S32);
+    int rowStart          = PS_SCALAR_VALUE(job->args->data[8],S32);
+    int rowStop           = PS_SCALAR_VALUE(job->args->data[9],S32);
+    return psImageSmoothMask_ScanRows_F32(calculation, calcMask, image, mask, maskVal,
+                                          gaussNorm, minGauss, size,
+                                          rowStart, rowStop);
 }
 
@@ -941,23 +941,23 @@
     psImage *calculation  = job->args->data[1]; // calculation image
     psImage *calcMask     = job->args->data[2]; // calculation mask
-    psMaskType maskVal 	  = PS_SCALAR_VALUE(job->args->data[3],U8);
-
-    psVector *gaussNorm	  = job->args->data[4]; // gauss kernel
-    float minGauss    	  = PS_SCALAR_VALUE(job->args->data[5],F32);
-    int size           	  = PS_SCALAR_VALUE(job->args->data[6],S32);
-    int colStart       	  = PS_SCALAR_VALUE(job->args->data[7],S32);
-    int colStop        	  = PS_SCALAR_VALUE(job->args->data[8],S32);
-    return psImageSmoothMask_ScanCols_F32(output, calculation, calcMask, maskVal, 
-					  gaussNorm, minGauss, size,
-					  colStart, colStop);
+    psMaskType maskVal    = PS_SCALAR_VALUE(job->args->data[3],U8);
+
+    psVector *gaussNorm   = job->args->data[4]; // gauss kernel
+    float minGauss        = PS_SCALAR_VALUE(job->args->data[5],F32);
+    int size              = PS_SCALAR_VALUE(job->args->data[6],S32);
+    int colStart          = PS_SCALAR_VALUE(job->args->data[7],S32);
+    int colStop           = PS_SCALAR_VALUE(job->args->data[8],S32);
+    return psImageSmoothMask_ScanCols_F32(output, calculation, calcMask, maskVal,
+                                          gaussNorm, minGauss, size,
+                                          colStart, colStop);
 }
 
 psImage *psImageSmoothMask_Threaded(psImage *output,
-				    const psImage *image,
-				    const psImage *mask,
-				    psMaskType maskVal,
-				    float sigma,
-				    float numSigma,
-				    float minGauss)
+                                    const psImage *image,
+                                    const psImage *mask,
+                                    psMaskType maskVal,
+                                    float sigma,
+                                    float numSigma,
+                                    float minGauss)
 {
     PS_ASSERT_IMAGE_NON_NULL(image, NULL);
@@ -993,84 +993,84 @@
 
     switch (image->type.type) {
-	// Smooth an image with masked pixels
-	// The calculation and calcMask images are *deliberately* backwards (row,col instead of col,row or
-	// [col][row] instead of [row][col]) to optimise the iteration over rows; such instances are marked "BW"
+        // Smooth an image with masked pixels
+        // The calculation and calcMask images are *deliberately* backwards (row,col instead of col,row or
+        // [col][row] instead of [row][col]) to optimise the iteration over rows; such instances are marked "BW"
 
       case PS_TYPE_F32: {
-	  psImage *calculation = psImageAlloc(numRows, numCols, PS_TYPE_F32); /* Calculation image; BW */
-	  psImage *calcMask = psImageAlloc(numRows, numCols, PS_TYPE_MASK); /* Mask for calculation image; BW */
-   
-	  /** Smooth in X direction **/
-	  for (int rowStart = 0; rowStart < numRows; rowStart+=scanRows) {
-	      int rowStop = PS_MIN (rowStart + scanRows, numRows);
-
-	      // allocate a job, construct the arguments for this job
-	      psThreadJob *job = psThreadJobAlloc("PSLIB_IMAGE_SMOOTHMASK_SCANROWS");
-	      psArrayAdd(job->args, 1, calculation);
-	      psArrayAdd(job->args, 1, calcMask);
-	      psArrayAdd(job->args, 1, (psImage *) image); // cast away const
-	      psArrayAdd(job->args, 1, (psImage *) mask); // cast away const
-	      PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_U8);
-	      psArrayAdd(job->args, 1, gaussNorm);
-	      PS_ARRAY_ADD_SCALAR(job->args, minGauss, PS_TYPE_F32);
-	      PS_ARRAY_ADD_SCALAR(job->args, size,     PS_TYPE_S32);
-	      PS_ARRAY_ADD_SCALAR(job->args, rowStart, PS_TYPE_S32);
-	      PS_ARRAY_ADD_SCALAR(job->args, rowStop,  PS_TYPE_S32);
-	      // -> psImageSmoothMask_ScanRows_F32 (calculation, calcMask, image, mask, maskVal, gauss, minGauss, size, rowStart, rowStop);
-
-	      // if threading is not active, we simply run the job and return
-	      if (!psThreadJobAddPending(job)) {
-		  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
-		  psFree(job);
-		  return false;
-	      }
-	      psFree(job);
-
-	  }
-
-	  // wait here for the threaded jobs to finish (NOP if threading is not active)
-	  if (!psThreadPoolWait(true)) {
-	      psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
-	      return false;
-	  }
-
-	  output = psImageRecycle(output, numCols, numRows, PS_TYPE_F32);
-   
-	  /** Smooth in Y direction  **/
-	  for (int colStart = 0; colStart < numCols; colStart+=scanCols) {
-	      int colStop = PS_MIN (colStart + scanCols, numCols);
-
-	      // allocate a job, construct the arguments for this job
-	      psThreadJob *job = psThreadJobAlloc("PSLIB_IMAGE_SMOOTHMASK_SCANCOLS");
-	      psArrayAdd(job->args, 1, output);
-	      psArrayAdd(job->args, 1, calculation);
-	      psArrayAdd(job->args, 1, calcMask);
-	      PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_U8);
-	      psArrayAdd(job->args, 1, gaussNorm);
-	      PS_ARRAY_ADD_SCALAR(job->args, minGauss, PS_TYPE_F32);
-	      PS_ARRAY_ADD_SCALAR(job->args, size,     PS_TYPE_S32);
-	      PS_ARRAY_ADD_SCALAR(job->args, colStart, PS_TYPE_S32);
-	      PS_ARRAY_ADD_SCALAR(job->args, colStop,  PS_TYPE_S32);
-	      // -> psImageSmoothMask_ScanCols_F32 (output, calculation, calcMask, maskVal, gauss, minGauss, size, colStart, colStop);
-
-	      // if threading is not active, we simply run the job and return
-	      if (!psThreadJobAddPending(job)) {
-		  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
-		  psFree(job);
-		  return false;
-	      }
-	      psFree(job);
-	  }
-
-	  // wait here for the threaded jobs to finish (NOP if threading is not active)
-	  if (!psThreadPoolWait(true)) {
-	      psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
-	      return false;
-	  }
-	  psFree(calculation);
-	  psFree(calcMask);
-	  psFree(gaussNorm);
-   
-	  return output;
+          psImage *calculation = psImageAlloc(numRows, numCols, PS_TYPE_F32); /* Calculation image; BW */
+          psImage *calcMask = psImageAlloc(numRows, numCols, PS_TYPE_MASK); /* Mask for calculation image; BW */
+
+          /** Smooth in X direction **/
+          for (int rowStart = 0; rowStart < numRows; rowStart+=scanRows) {
+              int rowStop = PS_MIN (rowStart + scanRows, numRows);
+
+              // allocate a job, construct the arguments for this job
+              psThreadJob *job = psThreadJobAlloc("PSLIB_IMAGE_SMOOTHMASK_SCANROWS");
+              psArrayAdd(job->args, 1, calculation);
+              psArrayAdd(job->args, 1, calcMask);
+              psArrayAdd(job->args, 1, (psImage *) image); // cast away const
+              psArrayAdd(job->args, 1, (psImage *) mask); // cast away const
+              PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_U8);
+              psArrayAdd(job->args, 1, gaussNorm);
+              PS_ARRAY_ADD_SCALAR(job->args, minGauss, PS_TYPE_F32);
+              PS_ARRAY_ADD_SCALAR(job->args, size,     PS_TYPE_S32);
+              PS_ARRAY_ADD_SCALAR(job->args, rowStart, PS_TYPE_S32);
+              PS_ARRAY_ADD_SCALAR(job->args, rowStop,  PS_TYPE_S32);
+              // -> psImageSmoothMask_ScanRows_F32 (calculation, calcMask, image, mask, maskVal, gauss, minGauss, size, rowStart, rowStop);
+
+              // if threading is not active, we simply run the job and return
+              if (!psThreadJobAddPending(job)) {
+                  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
+                  psFree(job);
+                  return false;
+              }
+              psFree(job);
+
+          }
+
+          // wait here for the threaded jobs to finish (NOP if threading is not active)
+          if (!psThreadPoolWait(true)) {
+              psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
+              return false;
+          }
+
+          output = psImageRecycle(output, numCols, numRows, PS_TYPE_F32);
+
+          /** Smooth in Y direction  **/
+          for (int colStart = 0; colStart < numCols; colStart+=scanCols) {
+              int colStop = PS_MIN (colStart + scanCols, numCols);
+
+              // allocate a job, construct the arguments for this job
+              psThreadJob *job = psThreadJobAlloc("PSLIB_IMAGE_SMOOTHMASK_SCANCOLS");
+              psArrayAdd(job->args, 1, output);
+              psArrayAdd(job->args, 1, calculation);
+              psArrayAdd(job->args, 1, calcMask);
+              PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_U8);
+              psArrayAdd(job->args, 1, gaussNorm);
+              PS_ARRAY_ADD_SCALAR(job->args, minGauss, PS_TYPE_F32);
+              PS_ARRAY_ADD_SCALAR(job->args, size,     PS_TYPE_S32);
+              PS_ARRAY_ADD_SCALAR(job->args, colStart, PS_TYPE_S32);
+              PS_ARRAY_ADD_SCALAR(job->args, colStop,  PS_TYPE_S32);
+              // -> psImageSmoothMask_ScanCols_F32 (output, calculation, calcMask, maskVal, gauss, minGauss, size, colStart, colStop);
+
+              // if threading is not active, we simply run the job and return
+              if (!psThreadJobAddPending(job)) {
+                  psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
+                  psFree(job);
+                  return false;
+              }
+              psFree(job);
+          }
+
+          // wait here for the threaded jobs to finish (NOP if threading is not active)
+          if (!psThreadPoolWait(true)) {
+              psError(PS_ERR_UNKNOWN, false, "Unable to smooth image");
+              return false;
+          }
+          psFree(calculation);
+          psFree(calcMask);
+          psFree(gaussNorm);
+
+          return output;
       }
       default:
@@ -1507,6 +1507,7 @@
 // XXX for now, either thread all or none
 // have to call this before calling psImageSmoothMask_Threaded
-void psImageConvolveSetThreads(bool set)
-{
+bool psImageConvolveSetThreads(bool set)
+{
+    bool old = threaded;                // Old value
     if (set && !threaded) {
         {
@@ -1534,4 +1535,5 @@
     }
     threaded = set;
+    return old;
 }
 
Index: branches/bills_081204/psLib/src/imageops/psImageConvolve.h
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageConvolve.h	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageConvolve.h	(revision 20890)
@@ -5,6 +5,6 @@
  * @author Robert DeSonia, MHPCC
  *
- * @version $Revision: 1.37 $ $Name: not supported by cvs2svn $
- * @date $Date: 2008-10-29 20:43:18 $
+ * @version $Revision: 1.38 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-26 00:43:12 $
  * Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
  */
@@ -209,10 +209,10 @@
 
 psImage *psImageSmoothMask_Threaded(psImage *output,
-				    const psImage *image,
-				    const psImage *mask,
-				    psMaskType maskVal,
-				    float sigma,
-				    float numSigma,
-				    float minGauss);
+                                    const psImage *image,
+                                    const psImage *mask,
+                                    psMaskType maskVal,
+                                    float sigma,
+                                    float numSigma,
+                                    float minGauss);
 
 bool psImageSmoothMaskF32(
@@ -225,5 +225,7 @@
 
 /// Control threading for image convolution functions
-void psImageConvolveSetThreads(bool threaded ///< Run image convolution threaded?
+///
+/// Returns old threading status
+bool psImageConvolveSetThreads(bool threaded ///< Run image convolution threaded?
     );
 
Index: branches/bills_081204/psLib/src/imageops/psImageInterpolate.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageInterpolate.c	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageInterpolate.c	(revision 20890)
@@ -7,6 +7,6 @@
  *  @author Paul Price, IfA
  *
- *  @version $Revision: 1.29 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-22 22:28:44 $
+ *  @version $Revision: 1.30 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-15 03:09:34 $
  *
  *  Copyright 2004-2007 Institute for Astronomy, University of Hawaii
@@ -427,5 +427,5 @@
 #define INTERPOLATE_RESULT() \
     psImageInterpolateStatus status = PS_INTERPOLATE_STATUS_ERROR; /* Status of interpolation */ \
-    *imageValue = sumImage / sumKernel; \
+    *imageValue = sumKernel > 0 ? sumImage / sumKernel : interp->badImage; \
     if (wantVariance) { \
         *varianceValue = sumVariance / (sumKernel2 - sumBad); \
Index: branches/bills_081204/psLib/src/imageops/psImageMap.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageMap.c	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageMap.c	(revision 20890)
@@ -7,6 +7,6 @@
  *  @author Eugene Magnier, IfA
  *
- *  @version $Revision: 1.10 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-09 20:42:51 $
+ *  @version $Revision: 1.11 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-17 02:37:28 $
  *
  *  Copyright 2007 Institute for Astronomy, University of Hawaii
@@ -349,6 +349,6 @@
 double psImageMapEval(const psImageMap *map, float x, float y)
 {
-    // This may be called in a type loop, so no assertions
-    return psImageUnbinPixel_V2(x, y, map->map, map->binning);
+    // This may be called in a tight loop, so no assertions
+    return psImageUnbinPixel(x, y, map->map, map->binning);
 }
 
@@ -365,5 +365,5 @@
 
     for (int i = 0; i < x->n; i++) {
-        result->data.F32[i] = psImageUnbinPixel_V2(x->data.F32[i], y->data.F32[i], map->map, map->binning);
+        result->data.F32[i] = psImageUnbinPixel(x->data.F32[i], y->data.F32[i], map->map, map->binning);
     }
 
Index: branches/bills_081204/psLib/src/imageops/psImageUnbin.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageUnbin.c	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageUnbin.c	(revision 20890)
@@ -7,6 +7,6 @@
  *  @author Eugene Magnier, IfA
  *
- *  @version $Revision: 1.8 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2007-09-20 23:55:50 $
+ *  @version $Revision: 1.9 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-17 02:38:43 $
  *
  *  Copyright 2007 Institute for Astronomy, University of Hawaii
@@ -24,4 +24,6 @@
 #include "psImageBinning.h"
 #include "psImageUnbin.h"
+
+# define UNBIN_TOL 1e-4
 
 // interpolate from model to background (~bresenham linear interpolation)
@@ -67,17 +69,41 @@
             // (Xs,Ys) : (Xe,Ye) : binned pixel centers in unbinned coords
             // corresponding to (Ix,Iy), (Ix+1,Iy+1)
-            int Xs = PS_MAX (0, PS_MIN (Nx, (Ix + 0.5)*DX - dx));
-            int Ys = PS_MAX (0, PS_MIN (Ny, (Iy + 0.5)*DY - dy));
+	    // XXX should this be "+ dx" and + dy?
+            int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, Ix + 0.5)));
+            int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, Iy + 0.5)));
             int Xe = PS_MAX (0, PS_MIN (Nx, Xs + DX));
             int Ye = PS_MAX (0, PS_MIN (Ny, Ys + DY));
 
             for (int iy = Ys; (iy < Ye) && (iy < Ny); iy++) {
-                float Vxs = (V10 - V00)*(iy - Ys) / DY + V00;
-                float Vxe = (V11 - V01)*(iy - Ys) / DY + V01;
+		float dY = (iy - Ys) / (float) DY;
+		float rY = 1.0 - dY;
+                float Vxs = V10*dY + V00*rY;
+                float Vxe = V11*dY + V01*rY;
+
+                // Vxs = (V10 - V00)*(iy - Ys) / DY + V00;                                                                                                       
+                // Vxe = (V11 - V01)*(iy - Ys) / DY + V01;                                                                                                       
+
+		// dVx = Vxs_1 - Vxs_1
+		// dVx = (V10*dY_1 + V00*rY_1) - (V10*dY_0 + V00*rY_0);
+		// dY_0 = (iy - Ys)/DY     = iy/DY - Ys/DY;
+		// dY_1 = (iy + 1 - Ys)/DY = iy/DY - Ys/DY + 1/DY;
+		// rY_0 = 1 - dY_0;
+		// rY_1 = 1 - dY_1;
+		// dVx = V10*(ddY) + V00*(drY);
+		// ddY = 1/DY;
+		// drY = -1/DY;
+
+		// dVxs = (V10 - V00)/DY;
+		// dVxe = (V11 - V01)/DY;
+
+		// ddV = (Vxe_1 - Vxs_1)/DX - (Vxe_0 - Vxs_0)/DX;
+		// ddV = (Vxe_1 - Vxe_0)/DX - (Vxs_1 - Vxs_0)/DX;
+		// ddV = (V11 - V01 - V10 + V00)/(DX*DY);
+
                 float dV = (Vxe - Vxs) / DX;
                 float V  = Vxs;
                 for (int ix = Xs; (ix < Xe) && (ix < Nx); ix++) {
                     vOut[iy][ix] = V;
-                    //assert (fabs(V - psImageUnbinPixel(ix, iy, in, DX, DY, dx, dy)) < 1e-5*fabs(V));
+                    // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V));
                     V += dV;
                 }
@@ -87,9 +113,9 @@
 
     // side pixels
-    int Xs = PS_MAX (0, PS_MIN (Nx, ( 0 + 0.5)*DX - dx));
-    int Xe = PS_MAX (0, PS_MIN (Nx, (nx - 0.5)*DX - dx));
+    int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, 0  + 0.5)));
+    int Xe = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, nx - 0.5)));
     for (int Iy = 0; Iy < ny - 1; Iy++) {
 
-        int Ys = PS_MAX (0, PS_MIN (Ny, (Iy + 0.5)*DY - dy));
+        int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, Iy + 0.5)));
         int Ye = PS_MAX (0, PS_MIN (Ny, Ys + DY));
 
@@ -102,4 +128,5 @@
             for (int ix = 0; ix < Xs; ix++) {
                 vOut[iy][ix] = V;
+		// assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V));
             }
             V += dV;
@@ -114,4 +141,5 @@
             for (int ix = Xe; ix < Nx; ix++) {
                 vOut[iy][ix] = V;
+		// assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V));
             }
             V += dV;
@@ -120,9 +148,9 @@
 
     // top and bottom pixels
-    int Ys = PS_MAX (0, PS_MIN (Ny, ( 0 + 0.5)*DY - dy));
-    int Ye = PS_MAX (0, PS_MIN (Ny, (ny - 0.5)*DY - dy));
+    int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, 0  + 0.5)));
+    int Ye = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, ny - 0.5)));
     for (int Ix = 0; Ix < nx - 1; Ix++) {
 
-        int Xs = PS_MAX (0, PS_MIN (Nx, (Ix + 0.5)*DX - dx));
+        int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, Ix + 0.5)));
         int Xe = PS_MAX (0, PS_MIN (Nx, Xs + DX));
 
@@ -135,4 +163,5 @@
             for (int ix = Xs; (ix < Xe) && (ix < Nx); ix++) {
                 vOut[iy][ix] = V;
+		// assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V));
                 V += dV;
             }
@@ -147,4 +176,5 @@
             for (int ix = Xs; (ix < Xe) && (ix < Nx); ix++) {
                 vOut[iy][ix] = V;
+		// assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V));
                 V += dV;
             }
@@ -157,11 +187,13 @@
         float V;
 	// center of last pixel 
-	int Xs = PS_MAX (0, PS_MIN (Nx, ( 0 + 0.5)*DX - dx));
-	int Xe = PS_MAX (0, PS_MIN (Nx, (nx - 0.5)*DX - dx));
-	int Ys = PS_MAX (0, PS_MIN (Ny, ( 0 + 0.5)*DY - dy));
-	int Ye = PS_MAX (0, PS_MIN (Ny, (ny - 0.5)*DY - dy));
+	int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, 0  + 0.5)));
+	int Xe = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, nx - 0.5)));
+	int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, 0  + 0.5)));
+	int Ye = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, ny - 0.5)));
 
         // 0,0
         V = vIn[0][0];
+	// assert (fabs(V - psImageUnbinPixel(0, 0, in, binning)) < UNBIN_TOL*fabs(V));
+
         for (int iy = 0; iy < Ys; iy++)
         {
@@ -172,4 +204,6 @@
         // Nx,0
         V = vIn[0][nx-1];
+	// assert (fabs(V - psImageUnbinPixel(Nx-1, 0, in, binning)) < UNBIN_TOL*fabs(V));
+
         for (int iy = 0; iy < Ys; iy++)
         {
@@ -180,4 +214,6 @@
         // 0,Ny
         V = vIn[ny-1][0];
+	// assert (fabs(V - psImageUnbinPixel(0, Ny-1, in, binning)) < UNBIN_TOL*fabs(V));
+
         for (int iy = Ye; iy < Ny; iy++)
         {
@@ -188,4 +224,6 @@
         // Nx,Ny
         V = vIn[ny-1][nx-1];
+	// assert (fabs(V - psImageUnbinPixel(Nx-1, Ny-1, in, binning)) < UNBIN_TOL*fabs(V));
+
         for (int iy = Ye; iy < Ny; iy++)
         {
@@ -202,8 +240,132 @@
 /*
  * Get the value of a single unbinned pixel from the binned representation
- *
+ */
+
+double psImageUnbinPixel(const double xFine, const double yFine, // desired Unbinned point (parent coords)
+			 const psImage *in, // binned image
+			 const psImageBinning *binning)   //!< Overhang
+{
+    PS_ASSERT_IMAGE_NON_NULL(in, NAN);
+    assert (in->type.type == PS_TYPE_F32);
+
+    const float xRuff = psImageBinningGetRuffX (binning, xFine);
+    const float yRuff = psImageBinningGetRuffY (binning, yFine);
+
+    const double value = psImageInterpolatePixelBilinear (xRuff, yRuff, in);
+
+    return value;
+}
+
+// fast & simple API to interpolate to a subpixel position using bilinear interpolation
+// x,y in parent image coordinates (pixel centers at 0.5, 0.5)
+double psImageInterpolatePixelBilinear (const double xIn, const double yIn, const psImage *in) {
+
+    PS_ASSERT_PTR_NON_NULL(in, PS_ERR_BAD_PARAMETER_VALUE);
+    assert (in->type.type == PS_TYPE_F32);
+
+    const double x = xIn - in->col0;
+    const double y = yIn - in->row0;
+
+    // allow extrapolation one pixel beyong edge of valid pixels, but no further
+    // (this allows the nXskip,nYskip boundary areas to be used as well)
+    if ((x < -1) || (x > in->numCols) || (y < -1) || (y > in->numRows)) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Point (%f,%f) lies outside binned image", x, y);
+        return NAN;
+    }
+
+    // limiting cases: Nx == 1 and/or Ny == 1
+
+    // if we have a single pixel, there is no spatial information
+    if ((in->numCols == 1) && (in->numRows == 1)) {
+	const double value = in->data.F32[0][0];
+	return value;
+    }
+
+    // handle edge cases with extrapolation
+
+    const int ix = x - 0.5; // index of reference pixel
+    const int iy = y - 0.5; // index of reference pixel
+
+    // do numCols,Rows first so we are never < 0
+    const int Xs = PS_MAX (PS_MIN (ix, in->numCols - 2), 0);
+    const int Ys = PS_MAX (PS_MIN (iy, in->numRows - 2), 0);
+
+    const int Xe = Xs + 1;
+    const int Ye = Ys + 1;
+
+    // dx,dy range from 0.0 to 1.0 for interpolated pixels, and -0.5 to 1.5 for extrapolation
+    const double dx = x - 0.5 - Xs;
+    const double dy = y - 0.5 - Ys;
+
+    const double rx = 1.0 - dx;
+    const double ry = 1.0 - dy;
+
+    // if Nx == 1, we have no x-dir spatial information
+    if (in->numCols == 1) {
+	double V0 = in->data.F32[Ys][Xs];
+	double V1 = in->data.F32[Ye][Xs];
+
+	const double value = V0*ry + V1*dy;
+	return value;
+    }	
+
+    // if Ny == 1, we have no y-dir spatial information
+    if (in->numRows == 1) {
+	double V0 = in->data.F32[Ys][Xs];
+	double V1 = in->data.F32[Ys][Xe];
+
+	const double value = V0*rx + V1*dx;
+	return value;
+    }	
+
+    // Vxy 
+    double V00 = in->data.F32[Ys][Xs];
+    double V01 = in->data.F32[Ye][Xs];
+    double V10 = in->data.F32[Ys][Xe];
+    double V11 = in->data.F32[Ye][Xe];
+
+    double value;
+
+    // corners
+    if ((dx < 0.0) && (dy < 0.0)) {
+	return V00;
+    }	
+    if ((dx > 1.0) && (dy < 0.0)) {
+	return V10;
+    }	
+    if ((dx < 0.0) && (dy > 1.0)) {
+	return V01;
+    }	
+    if ((dx > 1.0) && (dy > 1.0)) {
+	return V11;
+    }	
+
+    // sides
+    if (dx < 0.0) {
+	value = V00*ry + V01*dy;
+	return value;
+    }	
+    if (dy < 0.0) {
+	value = V00*rx + V10*dx;
+	return value;
+    }	
+    if (dx > 1.0) {
+	value = V10*ry + V11*dy;
+	return value;
+    }	
+    if (dy > 1.0) {
+	value = V01*rx + V11*dx;
+	return value;
+    }	
+
+    // bilinear interpolation
+    value = V00*rx*ry + V10*dx*ry + V01*rx*dy + V11*dx*dy;
+    return value;
+}
+
+# if (0)
+/* Old version of this function
  * N.b. This code only works for the central part of the image; the edge
  * cases should be added
-
  * XXXX this is bilinear interpolation, but written sub-optimally
  * XXXX this function should be taking float input coordinates!!!
@@ -253,91 +415,4 @@
 }
 
-double psImageUnbinPixel_V2(const double xFine, const double yFine, // desired Unbinned point (parent coords)
-			    const psImage *in, // binned image
-			    const psImageBinning *binning)   //!< Overhang
-{
-    PS_ASSERT_IMAGE_NON_NULL(in, NAN);
-    assert (in->type.type == PS_TYPE_F32);
-
-    const float xRuff = psImageBinningGetRuffX (binning, xFine);
-    const float yRuff = psImageBinningGetRuffY (binning, yFine);
-
-    const double value = psImageInterpolatePixelBilinear (xRuff, yRuff, in);
-
-    return value;
-}
-
-// fast & simple API to interpolate to a subpixel position using bilinear interpolation
-// x,y in parent image coordinates (pixel centers at 0.5, 0.5)
-double psImageInterpolatePixelBilinear (const double xIn, const double yIn, const psImage *in) {
-
-    PS_ASSERT_PTR_NON_NULL(in, PS_ERR_BAD_PARAMETER_VALUE);
-    assert (in->type.type == PS_TYPE_F32);
-
-    const double x = xIn - in->col0;
-    const double y = yIn - in->row0;
-
-    // allow extrapolation to edge of valid pixels, but not beyond
-    if ((x < 0) || (x >= in->numCols) || (y < 0) || (y >= in->numRows)) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Point (%f,%f) lies outside binned image", x, y);
-        return NAN;
-    }
-
-    // limiting cases: Nx == 1 and/or Ny == 1
-
-    // if we have a single pixel, there is no spatial information
-    if ((in->numCols == 1) && (in->numRows == 1)) {
-	const double value = in->data.F32[0][0];
-	return value;
-    }
-
-    // handle edge cases with extrapolation
-
-    const int ix = x - 0.5; // index of reference pixel
-    const int iy = y - 0.5; // index of reference pixel
-
-    // do numCols,Rows first so we are never < 0
-    const int Xs = PS_MAX (PS_MIN (ix, in->numCols - 2), 0);
-    const int Ys = PS_MAX (PS_MIN (iy, in->numRows - 2), 0);
-
-    const int Xe = Xs + 1;
-    const int Ye = Ys + 1;
-
-    // dx,dy range from 0.0 to 1.0 for interpolated pixels, and -0.5 to 1.5 for extrapolation
-    const double dx = x - 0.5 - Xs;
-    const double dy = y - 0.5 - Ys;
-
-    const double rx = 1.0 - dx;
-    const double ry = 1.0 - dy;
-
-    // if Nx == 1, we have no x-dir spatial information
-    if (in->numCols == 1) {
-	double V0 = in->data.F32[Ys][Xs];
-	double V1 = in->data.F32[Ye][Xs];
-
-	const double value = V0*ry + V1*dy;
-	return value;
-    }	
-
-    // if Ny == 1, we have no y-dir spatial information
-    if (in->numRows == 1) {
-	double V0 = in->data.F32[Ys][Xs];
-	double V1 = in->data.F32[Ys][Xe];
-
-	const double value = V0*rx + V1*dx;
-	return value;
-    }	
-
-    // Vxy 
-    double V00 = in->data.F32[Ys][Xs];
-    double V10 = in->data.F32[Ys][Xe];
-    double V01 = in->data.F32[Ye][Xs];
-    double V11 = in->data.F32[Ye][Xe];
-
-    // bilinear interpolation
-    const double value = V00*rx*ry + V10*dx*ry + V01*rx*dy + V11*dx*dy;
-
-    return value;
-}
-
-    
+# endif
+
Index: branches/bills_081204/psLib/src/imageops/psImageUnbin.h
===================================================================
--- branches/cnb_branch_20081104/psLib/src/imageops/psImageUnbin.h	(revision 20530)
+++ branches/bills_081204/psLib/src/imageops/psImageUnbin.h	(revision 20890)
@@ -6,6 +6,6 @@
  *  @author Robert DeSonia, MHPCC
  *
- *  $Revision: 1.5 $ $Name: not supported by cvs2svn $
- *  $Date: 2007-09-20 23:55:52 $
+ *  $Revision: 1.6 $ $Name: not supported by cvs2svn $
+ *  $Date: 2008-11-17 02:38:46 $
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
  */
@@ -23,10 +23,5 @@
     );
 
-double psImageUnbinPixel(const int ix, const int iy, //!< desired Unbinned point
-                         const psImage *in, //!< binned image
-			 const psImageBinning *binning ///< binning definition
-                        );
-
-double psImageUnbinPixel_V2(const double xFine, const double yFine, // desired Unbinned point (parent coords)
+double psImageUnbinPixel(const double xFine, const double yFine, // desired Unbinned point (parent coords)
 			    const psImage *in, // binned image
 			    const psImageBinning *binning   //!< Overhang
@@ -37,2 +32,11 @@
 /// @}
 #endif
+
+# if (0)
+/* old version */
+double psImageUnbinPixel(const int ix, const int iy, //!< desired Unbinned point
+                         const psImage *in, //!< binned image
+			 const psImageBinning *binning ///< binning definition
+                        );
+# endif
+
Index: branches/bills_081204/psLib/src/math/psRandom.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/math/psRandom.c	(revision 20530)
+++ branches/bills_081204/psLib/src/math/psRandom.c	(revision 20890)
@@ -10,6 +10,6 @@
 *  @author GLG, MHPCC
 *
-*  @version $Revision: 1.17 $ $Name: not supported by cvs2svn $
-*  @date $Date: 2007-11-16 00:41:07 $
+*  @version $Revision: 1.18 $ $Name: not supported by cvs2svn $
+*  @date $Date: 2008-11-05 11:12:39 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -50,5 +50,5 @@
     } else {
         // Read urandom to get seed
-        fread(&seedVal, sizeof(psU64),1,fd);
+        if (fread(&seedVal, sizeof(psU64),1,fd)) {;} // ignore return value
         // Close file
         fclose(fd);
Index: branches/bills_081204/psLib/src/mathtypes/psImage.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/mathtypes/psImage.c	(revision 20530)
+++ branches/bills_081204/psLib/src/mathtypes/psImage.c	(revision 20890)
@@ -9,6 +9,6 @@
  *  @author Ross Harman, MHPCC
  *
- *  @version $Revision: 1.131 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2007-11-05 23:42:46 $
+ *  @version $Revision: 1.132 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-05 11:12:39 $
  *
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -545,7 +545,7 @@
                      char *name)
 {
-    write(fd,"matrix: ",8);
-    write(fd,name,strlen(name));
-    write(fd,"\n",1);
+    if (write(fd,"matrix: ",8)) {;} //ignore return value
+    if (write(fd,name,strlen(name))) {;} //ignore return value
+    if (write(fd,"\n",1)) {;} //ignore return value
 
     char buffer[20];
@@ -555,9 +555,9 @@
         for (int i = 0; i < a->numCols; i++) {
             snprintf (buffer,20, "%f  ", p_psImageGetElementF64(a, i, j));
-            write(fd,buffer,strlen(buffer));
-        }
-        write(fd,"\n",1);
-    }
-    write(fd,"\n",1);
+            if (write(fd,buffer,strlen(buffer))) {;} //ignore return value
+        }
+        if (write(fd,"\n",1)) {;} //ignore return value
+    }
+    if (write(fd,"\n",1)) {;} //ignore return value
     return (true);
 }
Index: branches/bills_081204/psLib/src/mathtypes/psVector.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/mathtypes/psVector.c	(revision 20530)
+++ branches/bills_081204/psLib/src/mathtypes/psVector.c	(revision 20890)
@@ -10,6 +10,6 @@
 *  @author Joshua Hoblitt, University of Hawaii
 *
-*  @version $Revision: 1.103 $ $Name: not supported by cvs2svn $
-*  @date $Date: 2008-09-12 00:22:48 $
+*  @version $Revision: 1.104 $ $Name: not supported by cvs2svn $
+*  @date $Date: 2008-11-05 11:12:40 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -711,12 +711,12 @@
 
     sprintf (line, "vector: %s\n", name);
-    write (fd, line, strlen(line));
+    if (write(fd, line, strlen(line))) {;} //ignore return value
 
     for (int i = 0; i < a[0].n; i++) {
         sprintf (line, "%f\n", p_psVectorGetElementF64(a, i));
-        write (fd, line, strlen(line));
+        if (write(fd, line, strlen(line))) {;} //ignore return value
     }
     sprintf (line, "\n");
-    write (fd, line, strlen(line));
+    if (write(fd, line, strlen(line))) {;} //ignore return value
     return (true);
 }
Index: branches/bills_081204/psLib/src/sys/psLogMsg.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/sys/psLogMsg.c	(revision 20530)
+++ branches/bills_081204/psLib/src/sys/psLogMsg.c	(revision 20890)
@@ -11,6 +11,6 @@
  *  @author GLG, MHPCC
  *
- *  @version $Revision: 1.71 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-11-04 18:36:54 $
+ *  @version $Revision: 1.72 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-05 10:58:04 $
  *
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -315,5 +315,6 @@
     *head_ptr = '\0';
 
-    (void)write(logFD, head, strlen(head));
+    if (write(logFD, head, strlen(head))) {;} // ignore return value
+    
     if (logMsg) {
         psString msg = NULL;            // Message to print
@@ -324,12 +325,12 @@
         char *line = strtok_r(msg, "\n", &msgPtr);
         while (line) {
-            (void)write(logFD, "    ", 4);
-            (void)write(logFD, line, strlen(line));
-            (void)write(logFD, "\n", 1);
+            if(write(logFD, "    ", 4)) {;} // ignore return value
+            if(write(logFD, line, strlen(line))) {;} // ignore return value
+            if(write(logFD, "\n", 1)) {;} // ignore return value
             line = strtok_r(NULL, "\n", &msgPtr);
         }
         psFree(msg);
     } else {
-        (void)write(logFD, "\n", 1);
+        if(write(logFD, "\n", 1)) {;} // ignore return value
     }
 
Index: branches/bills_081204/psLib/src/sys/psThread.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/sys/psThread.c	(revision 20530)
+++ branches/bills_081204/psLib/src/sys/psThread.c	(revision 20890)
@@ -254,4 +254,16 @@
 }
 
+// Harvest jobs from the
+static void psThreadJobHarvest(void)
+{
+    psThreadLock();
+    psThreadJob *job;           // Job from done queue
+    while ((job = psThreadJobGetDone())) {
+        psFree(job);
+    }
+    psThreadUnlock();
+    return;
+}
+
 // call this function after you have added jobs to the queue and
 bool psThreadPoolWait(bool harvest)
@@ -259,4 +271,8 @@
     if (!pool || pool->n == 0) {
         // No threads initialised, so everything's done
+        // Ensure everything is harvested, if requested
+        if (harvest) {
+            psThreadJobHarvest();
+        }
         return true;
     }
@@ -273,10 +289,5 @@
         // Harvest jobs, if requested
         if (harvest) {
-            psThreadLock();
-            psThreadJob *job;           // Job from done queue
-            while ((job = psThreadJobGetDone())) {
-                psFree(job);
-            }
-            psThreadUnlock();
+            psThreadJobHarvest();
         }
 
@@ -294,4 +305,8 @@
             // Nothing in the queue
             psThreadUnlock();
+            // Ensure everything is harvested, if requested
+            if (harvest) {
+                psThreadJobHarvest();
+            }
             return true;
         }
Index: branches/bills_081204/psLib/src/sys/psTrace.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/sys/psTrace.c	(revision 20530)
+++ branches/bills_081204/psLib/src/sys/psTrace.c	(revision 20890)
@@ -9,6 +9,6 @@
  *  @author GLG, MHPCC
  *
- *  @version $Revision: 1.87 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-04-17 23:43:02 $
+ *  @version $Revision: 1.88 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-05 10:58:04 $
  *
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -486,9 +486,9 @@
     } else {
         if (comp->level == PS_DEFAULT_TRACE_LEVEL) {
-            sprintf(line,"%*s%-*s %d\n", depth, "", 20 - depth, comp->name, defLevel);
-            write (traceFD, line, strlen(line));
+	    sprintf(line,"%*s%-*s %d\n", depth, "", 20 - depth, comp->name, defLevel);
+	    if (write (traceFD, line, strlen(line))) {;} // ignore return value
         } else {
-            sprintf(line, "%*s%-*s %d\n", depth, "", 20 - depth, comp->name, comp->level);
-            write (traceFD, line, strlen(line));
+	    sprintf(line, "%*s%-*s %d\n", depth, "", 20 - depth, comp->name, comp->level);
+	    if (write (traceFD, line, strlen(line))) {;} // ignore return value
         }
     }
@@ -615,20 +615,20 @@
         *head_ptr = '\0';
 
-        write(traceFD, head, strlen(head));
+        if(write(traceFD, head, strlen(head)) ) {;} // ignore return value
 
         if (traceMsg) {
             // We indent each message one space for each level of the message.
             for (int i = 0; i < level; i++) {
-                write (traceFD, " ", 1);
+                if (write (traceFD, " ", 1)) {;} // ignore return value
             }
             psString line = NULL;       // Line to print
             psStringAppendV(&line, format, ap);
-            write (traceFD, line, strlen(line));
+            if (write (traceFD, line, strlen(line))) {;} // ignore return value
             if (line[strlen(line) - 1] != '\n') {
-                write(traceFD, "\n", 1);
+                if (write(traceFD, "\n", 1)) {;} // ignore return value
             }
             psFree(line);
         } else {
-            write(traceFD, "\n", 1);
+            if (write(traceFD, "\n", 1)) {;} // ignore return value
         }
 
Index: branches/bills_081204/psLib/src/xml/psXML.c
===================================================================
--- branches/cnb_branch_20081104/psLib/src/xml/psXML.c	(revision 20530)
+++ branches/bills_081204/psLib/src/xml/psXML.c	(revision 20890)
@@ -10,6 +10,6 @@
 *  @author David Robbins, MHPCC
 *
-*  @version $Revision: 1.48 $ $Name: not supported by cvs2svn $
-*  @date $Date: 2006-10-12 23:43:58 $
+*  @version $Revision: 1.49 $ $Name: not supported by cvs2svn $
+*  @date $Date: 2008-11-05 11:12:40 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -626,5 +626,5 @@
     if ( psXMLDocToMem(doc, buf) ) {
         n = strlen(buf);
-        write(fd, buf, n);
+        if (write(fd, buf, n)) {;} //ignore return value
     } else {
         psError(PS_ERR_LOCATION_INVALID, true, _("Buffer to small to store XML doc."));
Index: branches/bills_081204/psLib/test/imageops/tap_psImageMapFit.c
===================================================================
--- branches/cnb_branch_20081104/psLib/test/imageops/tap_psImageMapFit.c	(revision 20530)
+++ branches/bills_081204/psLib/test/imageops/tap_psImageMapFit.c	(revision 20890)
@@ -84,5 +84,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		if ((ix < 3) && (iy < 3)) {
 		    is_float (model->data.F32[iy][ix], NAN, "model matches expected NaN");
@@ -158,5 +158,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], 1e-5, "model matches inputs");
 	    }
@@ -240,5 +240,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], 1e-5, "model matches inputs");
 	    }
@@ -323,5 +323,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], 1e-5, "model matches inputs");
 	    }
@@ -389,5 +389,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], FLT_EPSILON, "model matches inputs");
 	    }
@@ -450,5 +450,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], FLT_EPSILON, "model matches inputs");
 	    }
@@ -531,5 +531,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], FLT_EPSILON, "model matches inputs");
 	    }
@@ -600,5 +600,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], 10*FLT_EPSILON, "model matches inputs");
 	    }
@@ -691,5 +691,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], 5*FLT_EPSILON, "model matches inputs");
 	    }
@@ -758,5 +758,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], FLT_EPSILON, "model matches inputs");
 	    }
@@ -819,5 +819,5 @@
 	for (int ix = 0; ix < model->numCols; ix++) {
 	    for (int iy = 0; iy < model->numRows; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 		is_float_tol (model->data.F32[iy][ix], field->data.F32[iy][ix], FLT_EPSILON, "model matches inputs");
 	    }
Index: branches/bills_081204/psLib/test/imageops/tap_psImageMapFit2.c
===================================================================
--- branches/cnb_branch_20081104/psLib/test/imageops/tap_psImageMapFit2.c	(revision 20530)
+++ branches/bills_081204/psLib/test/imageops/tap_psImageMapFit2.c	(revision 20890)
@@ -83,5 +83,5 @@
 	for (int ix = 0; ix < 1000; ix++) {
 	    for (int iy = 0; iy < 1000; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 	    }
 	}
@@ -168,5 +168,5 @@
 	for (int ix = 0; ix < 1000; ix++) {
 	    for (int iy = 0; iy < 1000; iy++) {
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 	    }
 	}
@@ -243,5 +243,5 @@
 		// to the wrong binned coordinate.  
 		// XXX fix edge cases
-		model->data.F32[iy][ix] = psImageUnbinPixel_V2 (ix + 0.5, iy + 0.5, map->map, map->binning);
+		model->data.F32[iy][ix] = psImageUnbinPixel (ix + 0.5, iy + 0.5, map->map, map->binning);
 	    }
 	}
Index: branches/bills_081204/psModules/src/astrom/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/psModules/src/astrom/Makefile.am	(revision 20530)
+++ branches/bills_081204/psModules/src/astrom/Makefile.am	(revision 20890)
@@ -10,5 +10,6 @@
 	pmAstrometryModel.c \
 	pmAstrometryRefstars.c \
-	pmAstrometryWCS.c
+	pmAstrometryWCS.c \
+	pmAstrometryVisual.c 
 
 pkginclude_HEADERS = \
@@ -19,5 +20,6 @@
 	pmAstrometryModel.h \
 	pmAstrometryRefstars.h \
-	pmAstrometryWCS.h
+	pmAstrometryWCS.h \
+	pmAstrometryVisual.h
 
 CLEANFILES = *~
Index: branches/bills_081204/psModules/src/astrom/pmAstrometryObjects.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/astrom/pmAstrometryObjects.c	(revision 20530)
+++ branches/bills_081204/psModules/src/astrom/pmAstrometryObjects.c	(revision 20890)
@@ -8,6 +8,6 @@
 *  @author EAM, IfA
 *
-*  @version $Revision: 1.41 $ $Name: not supported by cvs2svn $
-*  @date $Date: 2008-10-10 02:33:35 $
+*  @version $Revision: 1.42 $ $Name: not supported by cvs2svn $
+*  @date $Date: 2008-11-20 01:26:07 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
@@ -29,5 +29,4 @@
 #include <unistd.h>   // for unlink
 #include <pslib.h>
-#include <kapa.h> // cnb: do I need an IFDEF?
 
 #include "pmHDU.h"
@@ -35,4 +34,5 @@
 #include "pmAstrometryObjects.h"
 #include "pmKapaPlots.h"
+#include "pmAstrometryVisual.h"
 
 #define PM_ASTROMETRYOBJECTS_DEBUG 1
@@ -120,5 +120,4 @@
         i++;
     }
-
     return (matches);
 }
@@ -564,10 +563,4 @@
     psF32 **D2 = gridD2->data.F32;
 
-#ifdef PLOTS
-    // vectors to hold dX and dY
-    int nplot = raw->n * ref->n;
-    float dXplot[nplot];
-    float dYplot[nplot];
-#endif
 
     // accumulate grids for focal plane (L,M) matches
@@ -579,9 +572,4 @@
             dY = ob1->FP->y - ob2->FP->y;
 
-#ifdef PLOTS
-            dXplot[(i * ref->n) + j] = dX;
-            dYplot[(i * ref->n) + j] = dY;
-            // fprintf (f, "dX,dY: %8.2f %8.2f : %8.2f %8.2f : %8.2f %8.2f\n", dX, dY, ob1->FP->x, ob2->FP->x, ob1->FP->y, ob2->FP->y);
-#endif
             // find bin coordinates for this delta-delta
             if (!AstromGridBin (&iX, &iY, dX, dY)) {
@@ -618,5 +606,5 @@
         }
 
-        # if 0
+# if 0
         char line[16];
         psFits *fits = psFitsOpen ("grid.image.fits", "w");
@@ -625,5 +613,5 @@
         fprintf (stderr, "wrote grid image, press return to continue\n");
         fgets (line, 15, stdin);
-        # endif
+# endif
 
         // only check bins with at least 1/2 of max bin
@@ -670,4 +658,7 @@
         }
 
+        pmAstromVisualPlotGridMatch(raw, ref, gridNP, stats->offset.x, stats->offset.y,
+                                    maxOffpix, Scale, Offset);
+
         psFree (imStats);
         // XXX EAM : This routine, and pmAstromGridMatch, need to handle failure cases better
@@ -689,59 +680,4 @@
     // fprintf (stderr, "sigma: nMatch: %d, nTest: %d, nTen: %d\n", stats->nMatch, stats->nTest, sort->data.U32[sort->n - 10]);
 
-#ifdef PLOTS
-    /*** plot DXs and DYs ***/
-    KapaSection section = {"s1", 0.00, 0.00, 1.0, 1.0};
-    //KapaSection left = {"s2", 0.05, 0.05, 0.35, 0.8};
-    //KapaSection right = {"s3", 0.55, 0.05, 0.35, 0.8};
-    //KapaSection yprof = {"s4", 0.45, 0.05, 0.10, 0.9};
-    //KapaSection xprof = {"s5", .05, .90, .35, .1};
-
-    Graphdata graphdata;
-    int kapa = pmKapaOpen(true);
-    KapaClearPlots(kapa);
-    KapaInitGraph(&graphdata);
-    KapaSetSection(kapa, &section);
-    graphdata.xmin = stats->offset.x - 1.5 * maxOffpix;
-    graphdata.xmax = stats->offset.x + 1.5 * maxOffpix;
-    graphdata.ymin = stats->offset.y - 1.5 * maxOffpix;
-    graphdata.ymax = stats->offset.y + 1.5 * maxOffpix;
-
-    KapaSetLimits(kapa, &graphdata);
-    KapaSetFont(kapa, "helvetica", 14);
-    KapaBox(kapa, &graphdata);
-    KapaSendLabel (kapa, "X offset", KAPA_LABEL_XM);
-    KapaSendLabel (kapa, "Y offset", KAPA_LABEL_YM);
-    KapaSendLabel (kapa, "pmAstromGridAngle residuals. Big Box: Serach Region. Small Box: Correlation Peak.",
-                   KAPA_LABEL_XP);
-    graphdata.style = 2;
-    graphdata.ptype = 0;
-    graphdata.size = 0.4;
-    graphdata.color = KapaColorByName ("black");
-    KapaPrepPlot(kapa, nplot, &graphdata);
-    KapaPlotVector (kapa, nplot, dXplot, "x");
-    KapaPlotVector (kapa, nplot, dYplot, "y");
-
-    //Overplot bounding box, peak of distribution
-    float xbound[5] = { -maxOffpix, maxOffpix, maxOffpix, -maxOffpix, -maxOffpix};
-    float ybound[5] = { -maxOffpix, -maxOffpix, maxOffpix, maxOffpix, -maxOffpix};
-    float xbin[5] = {stats->offset.x - 0.5 * Scale, stats->offset.x + 0.5 * Scale, stats->offset.x + 0.5 * Scale,
-                     stats->offset.x - 0.5 * Scale, stats->offset.x - 0.5 * Scale};
-    float ybin[5] = {stats->offset.y - 0.5 * Scale, stats->offset.y - 0.5 * Scale, stats->offset.y + 0.5 * Scale,
-                     stats->offset.y + 0.5 * Scale, stats->offset.y - 0.5 * Scale};
-    graphdata.color = KapaColorByName("red");
-    graphdata.style = 0;
-    graphdata.size = 1.0;
-    KapaPrepPlot(kapa, 5, &graphdata);
-    KapaPlotVector (kapa, 5, xbound, "x");
-    KapaPlotVector (kapa, 5, ybound, "y");
-    KapaPrepPlot(kapa, 5, &graphdata);
-    KapaPlotVector (kapa, 5, xbin, "x");
-    KapaPlotVector (kapa, 5, ybin, "y");
-
-    KapaPNG(kapa, "dXdY.png");
-    char c;
-    fscanf(stdin, "%c", &c);
-    /*** end ***/
-#endif
 
   psFree (sort);
@@ -767,5 +703,4 @@
     PS_ASSERT_PTR_NON_NULL(ref, NULL);
     PS_ASSERT_PTR_NON_NULL(config, NULL);
-
     bool status;
     double xMin, xMax, yMin, yMax;
@@ -911,8 +846,4 @@
     psVector *xHistNew = psVectorSmooth(NULL, xHist, tweakSmooth, tweakNsigma);
     psVector *yHistNew = psVectorSmooth(NULL, yHist, tweakSmooth, tweakNsigma);
-    psFree(xHist);
-    psFree(yHist);
-    xHist = xHistNew;
-    yHist = yHistNew;
 
     // select peak in x and in y
@@ -921,11 +852,11 @@
     double yMax = 0;
     for (int i = 0; i < nBin; i++) {
-        if (xHist->data.F32[i] > xMax) {
+        if (xHistNew->data.F32[i] > xMax) {
             xBin = i;
-            xMax = xHist->data.F32[i];
-        }
-        if (yHist->data.F32[i] > yMax) {
+            xMax = xHistNew->data.F32[i];
+        }
+        if (yHistNew->data.F32[i] > yMax) {
             yBin = i;
-            yMax = yHist->data.F32[i];
+            yMax = yHistNew->data.F32[i];
         }
     }
@@ -940,7 +871,11 @@
     tweak->offset.y += yPeak;
 
+    pmAstromVisualPlotTweak (xHistNew, yHistNew, xBin, yBin);
+
     psFree (rot);
     psFree (xHist);
     psFree (yHist);
+    psFree (xHistNew);
+    psFree (yHistNew);
 
     return tweak;
Index: branches/bills_081204/psModules/src/astrom/pmAstrometryObjects.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/astrom/pmAstrometryObjects.h	(revision 20530)
+++ branches/bills_081204/psModules/src/astrom/pmAstrometryObjects.h	(revision 20890)
@@ -4,6 +4,6 @@
  * @author EAM, IfA
  *
- * @version $Revision: 1.17 $ $Name: not supported by cvs2svn $
- * @date $Date: 2007-11-21 07:02:55 $
+ * @version $Revision: 1.18 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-20 01:26:07 $
  * Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
  */
@@ -238,11 +238,9 @@
  *
  */
-bool pmAstromFitFPA(
-    pmFPA *fpa,
-    psArray *st1,
-    psArray *st2,
-    psArray *match,
-    psMetadata *config
-);
+bool pmAstromFitFPA(pmFPA *fpa,
+                    psArray *st1,
+                    psArray *st2,
+                    psArray *match,
+                    psMetadata *config);
 
 
@@ -269,6 +267,5 @@
     psArray *st2,
     psArray *match,
-    psMetadata *config
-);
+    psMetadata *config);
 
 
Index: branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.c
===================================================================
--- branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.c	(revision 20890)
+++ branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.c	(revision 20890)
@@ -0,0 +1,448 @@
+/** Diagnostic plots called from within pmAstrometry routines.
+ * @author      Chris Beaumont
+ * @date        October 2008
+ */
+
+/* Include Files   */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <strings.h>
+#include <string.h>
+#include <math.h>
+#include <assert.h>
+#include <pslib.h>
+
+#include "pmKapaPlots.h"
+
+#include "pmHDU.h"
+#include "pmFPA.h"
+#include "pmAstrometryObjects.h"
+
+# if (HAVE_KAPA)
+# include <kapa.h>
+
+# define KAPAX 700
+# define KAPAY 700
+
+
+//variables to determine when things are plotted
+static bool isVisual             = false;
+static bool plotGridMatch        = true;
+static bool plotTweak            = true;
+
+// variables to store plotting window indices
+static int kapa = -1;
+
+
+/* Utility Routine Prototypes */
+bool pmAstromVisualScaleGraphdata(Graphdata *graphdata, psVector *xVec, psVector *yVec, bool clip);
+
+
+/* Initialization Routines  */
+
+
+/** start or stop plotting */
+bool pmAstromSetVisual (bool mode) {
+    isVisual = mode;
+    return true;
+}
+
+
+/** open name, and resize a window if necessary */
+bool pmAstromVisualInitWindow (int *kapid, char *name) {
+    if (*kapid == -1) {
+        *kapid = KapaOpenNamedSocket("kapa", name);
+        if (*kapid == -1) {
+            fprintf (stderr, "failure to open kapa; visual mode disabled for pmAstrom\n");
+            isVisual = false;
+            return false;
+        }
+        KapaResize (*kapid, KAPAX, KAPAY);
+    }
+    return true;
+}
+
+
+/** ask the user how to proceed */
+bool pmAstromVisualAskUser(bool *plotflag)
+{
+    char key[10];
+    fprintf (stdout, "[c]ontinue? [s]kip the rest of these plots? [a]bort all visual plots?");
+    if (!fgets(key, 8, stdin)) {
+        psWarning("Unable to read option");
+    }
+    if (key[0] == 's') {
+        *plotflag = false;
+    }
+    if (key[0] == 'a') {
+        pmAstromSetVisual(false);
+    }
+    return true;
+}
+
+
+/** destroy windows at the end of a run*/
+bool pmAstromVisualClose()
+{
+    if(kapa != -1)
+        KiiClose(kapa);
+    return true;
+}
+
+
+/* plotting routines */
+
+/**
+ * Plot the offset between every pair of reference and raw source locations. The peak of this
+ * distribution nominally gives the offset, scale difference, and rotation of the two catalogs.
+ * Overplots the location of this peak as determined by pmAstromGridAngle, as well as some profiles
+ * along horizontal and vertical cuts through this peak.
+ */
+bool pmAstromVisualPlotGridMatch (const psArray *raw, ///< raw stars
+                                  const psArray *ref, ///< reference stars
+                                  psImage *gridNP,    ///< a 2D histogram of raw-ref star distances
+                                  double offsetX,     ///< The X location (FP coordinates) of the peak of gridNP
+                                  double offsetY,     ///< the Y location (FP coordinates) of the peak of gridNP
+                                  double maxOffpix,   ///< The half-width of gridNP in FP coordinates
+                                  double Scale,       ///< The pixel size of gridNP in histogram-bin-coordinates
+                                  double Offset       ///< The (x,y) location (histogram-bin coordinates) of the FP point (0,0) in gridNP
+                                  )
+{
+    //make sure we want to plot this
+    if (!isVisual || !plotGridMatch) return true;
+    if (!pmAstromVisualInitWindow(&kapa, "pmAstrom:plots"))
+        return false;
+
+    KapaSection section = {"s1", 0.05, 0.05, .75, .75};
+    KapaSection sectionY = {"s2", 0.8, 0.05, .15, .75};
+    KapaSection sectionX = {"s3", .05, .8, .75, .15};
+
+    Graphdata graphdata;
+    int nplot = raw->n * ref->n;                      // number of points to plot
+    float dXplot[nplot];                              // x data points
+    float dYplot[nplot];                              // y data points
+    pmAstromObj *ob1; pmAstromObj *ob2;               // shortcuts to the data in raw and ref
+    psU32 **NP = gridNP->data.U32;                    // shortcut to the gridNP data
+    float vertHistSlice[gridNP->numRows];             // vertical histogram slice through peak
+    float horizHistSlice[gridNP->numCols];            // horizontal histogram slice through peak
+    float horizontalIndices[gridNP->numCols];         // the horizontal offset corresponding to each bin of horizHistSlice
+    float verticalIndices[gridNP->numRows];           // the vertical offset corresponding to each bin of vertHistSlice
+    int maxHorizontalSlice = 0;                       // peak value of horizHistSlice
+    int maxVerticalSlice = 0;                         // peak value of vertHistSlice
+    int peakXbin = (int) (offsetX / Scale + Offset);  // X bin index of peak
+    int peakYbin = (int) (offsetY / Scale + Offset);  // Y bin index of peak
+
+
+    // set up plot information
+    KapaClearPlots(kapa);
+    KapaInitGraph(&graphdata);
+    KapaSetSection(kapa, &section);
+
+    graphdata.xmin = -1.0 * maxOffpix;
+    graphdata.xmax =  1.0 * maxOffpix;
+    graphdata.ymin = -1.0 * maxOffpix;
+    graphdata.ymax =  1.0 * maxOffpix;
+    KapaSetLimits(kapa, &graphdata);
+
+    KapaSetFont(kapa, "helvetica", 14);
+    KapaBox(kapa, &graphdata);
+    KapaSendLabel (kapa, "X offset (FP)", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Y offset (FP)", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "pmAstromGridAngle residuals. Box: Correlation Peak.",
+                   KAPA_LABEL_XP);
+    graphdata.style = 2;
+    graphdata.ptype = 0;
+    graphdata.size = 0.4;
+    graphdata.color = KapaColorByName ("black");
+
+    // calculate the plot points
+    float dX, dY;
+    for (int i = 0; i < raw->n; i++) {
+        ob1 = (pmAstromObj *)raw->data[i];
+        for (int j = 0; j < ref->n; j++) {
+            ob2 = (pmAstromObj *)ref->data[j];
+            dX = ob1->FP->x - ob2->FP->x;
+            dY = ob1->FP->y - ob2->FP->y;
+            dXplot[(i * ref->n) + j] = dX;
+            dYplot[(i * ref->n) + j] = dY;
+        }
+    }
+
+    // calculate the points for the profiles
+    for (int i = 0; i < gridNP->numRows; i++) {
+        vertHistSlice[i] = NP[i][peakXbin];
+        verticalIndices[i] = (i - Offset) * Scale;
+        if (vertHistSlice[i] > maxVerticalSlice) {
+            maxVerticalSlice = vertHistSlice[i];
+        }
+    }
+    for (int i = 0; i < gridNP->numCols; i++) {
+        horizHistSlice[i] = NP[peakYbin][i];
+        horizontalIndices[i] = (i - Offset) * Scale;
+        if (horizHistSlice[i] > maxHorizontalSlice) {
+            maxHorizontalSlice = horizHistSlice[i];
+        }
+    }
+
+    //Plot the offsets
+    KapaPrepPlot(kapa, nplot, &graphdata);
+    KapaPlotVector (kapa, nplot, dXplot, "x");
+    KapaPlotVector (kapa, nplot, dYplot, "y");
+
+    //Overplot bounding box, peak of distribution
+    float xbound[5] = { -maxOffpix, maxOffpix, maxOffpix, -maxOffpix, -maxOffpix};
+    float ybound[5] = { -maxOffpix, -maxOffpix, maxOffpix, maxOffpix, -maxOffpix};
+    float xbin[5] = {offsetX - 0.5 * Scale, offsetX + 0.5 * Scale,
+                     offsetX + 0.5 * Scale, offsetX - 0.5 * Scale,
+                     offsetX - 0.5 * Scale};
+    float ybin[5] = {offsetY - 0.5 * Scale, offsetY - 0.5 * Scale,
+                     offsetY + 0.5 * Scale, offsetY + 0.5 * Scale,
+                     offsetY - 0.5 * Scale};
+    graphdata.color = KapaColorByName("red");
+    graphdata.style = 0;
+    graphdata.size = 1.0;
+    KapaPrepPlot(kapa, 5, &graphdata);
+    KapaPlotVector (kapa, 5, xbound, "x");
+    KapaPlotVector (kapa, 5, ybound, "y");
+    KapaPrepPlot(kapa, 5, &graphdata);
+    KapaPlotVector (kapa, 5, xbin, "x");
+    KapaPlotVector (kapa, 5, ybin, "y");
+
+    //plot X profile
+    KapaSetSection(kapa, &sectionX);
+    graphdata.color = KapaColorByName("black");
+    graphdata.ptype = 1;
+    graphdata.style = 1;
+    graphdata.ymin = 0;
+    graphdata.ymax = maxHorizontalSlice + 0.5;
+    KapaSetLimits(kapa, &graphdata);
+
+    KapaBox(kapa, &graphdata);
+    KapaPrepPlot(kapa, gridNP->numCols, &graphdata);
+    KapaPlotVector (kapa, gridNP->numCols, horizontalIndices, "x");
+    KapaPlotVector (kapa, gridNP->numCols, horizHistSlice, "y");
+    float xslice[2] = {offsetX - Scale / 2., offsetX - Scale / 2.};
+    float yslice[2] = {-5, 100};
+    graphdata.color = KapaColorByName("red");
+    KapaPrepPlot(kapa, 2, &graphdata);
+    KapaPlotVector (kapa, 2, xslice, "x");
+    KapaPlotVector (kapa, 2, yslice, "y");
+
+    //plot Y profile
+    KapaSetSection(kapa, &sectionY);
+    graphdata.color = KapaColorByName("black");
+    graphdata.ptype = 1;
+    graphdata.style = 1;
+    graphdata.ymin = -maxOffpix;
+    graphdata.ymax = maxOffpix;
+    graphdata.xmin = -1.0 ;
+    graphdata.xmax = maxVerticalSlice + 0.5;
+    KapaSetLimits(kapa, &graphdata);
+
+    KapaBox(kapa, &graphdata);
+    KapaPrepPlot(kapa, gridNP->numRows, &graphdata);
+    KapaPlotVector (kapa, gridNP->numRows, vertHistSlice, "x");
+    KapaPlotVector (kapa, gridNP->numRows, verticalIndices, "y");
+    yslice[0] = yslice[1] = offsetY - Scale / 2.;
+    xslice[0] = -5; xslice[1] = 100;
+    graphdata.color = KapaColorByName("red");
+    KapaPrepPlot(kapa, 2, &graphdata);
+    KapaPlotVector (kapa, 2, xslice, "x");
+    KapaPlotVector (kapa, 2, yslice, "y");
+
+    pmAstromVisualAskUser(&plotGridMatch);
+    return true;
+} // end of pmAstromVisualPlotGridMatch
+
+
+/** Plot the refinements made within pmAstromGridTweak.
+ * After pmAstromGridMatch finds the best rotaion/scale/offset between raw and reference stars
+ * within a coarse grid of rotations/scales, pmAstromGridTweak computes a higher precision
+ * estimate of the offset. It computes the 2 point correlation function between raw and ref
+ * stars along horizontal and vertical cuts through the first-guess offset. It finds the peak
+ * of these two profiles and adjusts the offset accordingly. This procedure plots the profiles.
+ */
+bool pmAstromVisualPlotTweak (psVector *xHist, ///< Smoothed Horizontal cut through the histogram
+                              psVector *yHist, ///< Smoothed Vertical cut throug the histogram
+                              int xBin,        ///< X Bin index of the histogram peak
+                              int yBin         ///< Y bin index of the histogram peak
+    )
+{
+    //make sure we want to plot this
+    if (!isVisual || !plotTweak) return true;
+    if (!pmAstromVisualInitWindow(&kapa, "pmAstrom:plots")) {
+        return false;
+    }
+
+    Graphdata graphdata;
+    KapaSection section1 = {"s1", 0.05, 0.05, 0.90, 0.4};
+    KapaSection section2 = {"s2", 0.05, 0.5, 0.90, 0.4};
+
+    psVector *xIndices = psVectorAlloc (xHist->n, PS_TYPE_F32);
+    psVector *yIndices = psVectorAlloc (yHist->n, PS_TYPE_F32);
+
+    //populate the Indices vectors
+    for(int i = 0; i < xHist->n; i++) {
+        xIndices->data.F32[i] = i;
+    }
+    for(int i = 0; i < yHist->n; i++) {
+        yIndices->data.F32[i] = i;
+    }
+
+    // set up plot information
+    KapaClearPlots(kapa);
+    KapaInitGraph(&graphdata);
+
+    // plot the X histogram
+    pmAstromVisualScaleGraphdata(&graphdata, xIndices, xHist, false);
+    KapaSetSection(kapa, &section1);
+    KapaSetLimits (kapa, &graphdata);
+    KapaSetFont(kapa, "helvetica", 14);
+    KapaBox(kapa, &graphdata);
+    KapaSendLabel (kapa, "X offset Bin", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Number of Sources", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Horizontal Profile",
+                   KAPA_LABEL_XP);
+    graphdata.style = 1;
+    graphdata.ptype = 0;
+    graphdata.size = 0.4;
+    graphdata.color = KapaColorByName ("black");
+
+    KapaPrepPlot (kapa, xHist->n, &graphdata);
+    KapaPlotVector (kapa, xHist->n, xIndices->data.F32, "x");
+    KapaPlotVector (kapa, xHist->n, xHist->data.F32, "y");
+
+    //overplot the peak
+    float x[2] = {xBin, xBin};
+    float y[2] = {-500, 500};
+    graphdata.color = KapaColorByName ("red");
+    KapaPrepPlot (kapa, 2, &graphdata);
+    KapaPlotVector (kapa, 2, x, "x");
+    KapaPlotVector (kapa, 2, y, "y");
+
+    //plot the Y histogram
+    pmAstromVisualScaleGraphdata(&graphdata, yIndices, yHist, false);
+    KapaSetSection(kapa, &section2);
+    KapaSetLimits (kapa, &graphdata);
+    KapaSetFont(kapa, "helvetica", 14);
+    graphdata.color = KapaColorByName ("black");
+    KapaBox(kapa, &graphdata);
+    KapaSendLabel (kapa, "Y offset Bin", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Number of Sources", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Vertical Profile",
+                   KAPA_LABEL_XP);
+    graphdata.style = 1;
+    graphdata.ptype = 0;
+    graphdata.size = 0.4;
+
+    KapaPrepPlot (kapa, yHist->n, &graphdata);
+    KapaPlotVector (kapa, yHist->n, yIndices->data.F32, "x");
+    KapaPlotVector (kapa, yHist->n, yHist->data.F32, "y");
+
+    //overplot the peak
+    x[0] = x[1] = yBin;
+    graphdata.color = KapaColorByName ("red");
+    KapaPrepPlot (kapa, 2, &graphdata);
+    KapaPlotVector (kapa, 2, x, "x");
+    KapaPlotVector (kapa, 2, y, "y");
+
+    //plot title
+    graphdata.color = KapaColorByName("black");
+    KapaSection section3 = {"s3", 0, 0, 1, 1};
+    KapaSetSection( kapa, &section3);
+    KapaSendLabel (kapa, "Tweaking the Astrometry Grid Solution. Smoothed profiles + peak location",
+                   KAPA_LABEL_XP);
+
+    pmAstromVisualAskUser(&plotTweak);
+
+    psFree(xIndices);
+    psFree(yIndices);
+    return true;
+} //end of pmAstromPlotTweak
+
+
+/** pmAstromScaleGraphdata
+ * Scale the graphdata structure based on x and y coordinates. Use sigma clipping to
+ * prevent outliers from making te plot region too big.
+ */
+bool pmAstromVisualScaleGraphdata(Graphdata *graphdata, psVector *xVec, psVector *yVec, bool clip) {
+
+    graphdata->xmin = +FLT_MAX;
+    graphdata->xmax = -FLT_MAX;
+    graphdata->ymin = +FLT_MAX;
+    graphdata->ymax = -FLT_MAX;
+
+    //determine standard deviation of xVec and yVec
+    psStats *statsX = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
+    psStats *statsY = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
+    psVectorStats (statsX, xVec, NULL, NULL, 0);
+    psVectorStats (statsY, yVec, NULL, NULL, 0);
+
+    float xhi  = statsX->sampleMedian + 3 *statsX->sampleStdev;
+    float xlo = statsX->sampleMedian - 3 *statsX->sampleStdev;
+    float yhi = statsY->sampleMedian + 3 *statsY->sampleStdev;
+    float ylo = statsY->sampleMedian - 3 *statsY->sampleStdev;
+
+    // don't sigma clip
+    if (!clip) {
+        xhi = +FLT_MAX;
+        xlo = -FLT_MAX;
+        yhi = +FLT_MAX;
+        ylo = -FLT_MAX;
+    }
+
+    // abort if there is no good data
+    if (!isfinite(xhi) || !isfinite(xlo) || !isfinite(yhi) || !isfinite(ylo)) {
+        graphdata->xmin = -1;
+        graphdata->ymin  = -1;
+        graphdata->xmax = 1;
+        graphdata->ymax = 1;
+        psFree(statsX);
+        psFree(statsY);
+        return false;
+    }
+
+    for(int i = 0; i < xVec->n; i++) {
+        if (!isfinite(xVec->data.F32[i])) continue;
+        if (xVec->data.F32[i] > xhi || xVec->data.F32[i] < xlo) continue;
+        graphdata->xmin = PS_MIN (graphdata->xmin, xVec->data.F32[i]);
+        graphdata->xmax = PS_MAX (graphdata->xmax, xVec->data.F32[i]);
+    }
+
+    for (int i = 0; i < yVec->n; i++) {
+        if (!isfinite(xVec->data.F32[i])) continue;
+        if (yVec->data.F32[i] > yhi || yVec->data.F32[i] < ylo) continue;
+        graphdata->ymin = PS_MIN (graphdata->ymin, yVec->data.F32[i]);
+        graphdata->ymax = PS_MAX (graphdata->ymax, yVec->data.F32[i]);
+    }
+
+    // add a whitespace border
+    float range = graphdata->xmax - graphdata->xmin;
+    if (range == 0) range = 1;
+    graphdata->xmin -= .05 * range;
+    graphdata->xmax += .05 * range;
+
+    range = graphdata->ymax - graphdata->ymin;
+    if (range == 0) range = 1;
+    graphdata->ymin -= .05 * range;
+    graphdata->ymax += .05 * range;
+
+    psFree (statsX);
+    psFree (statsY);
+    return true;
+}
+
+
+# else
+
+bool pmAstromSetVisual(bool mode) { return true; }
+bool pmAstromVisualInitWindow (int *kapid, char *name) { return true; }
+bool pmAstromVisualAskUser (bool *plotflag) { return true; }
+bool pmAstromVisualClose() { return true; }
+bool pmAstromVisualPlotGridMatch (const psArray *raw, const psArray *ref, psImage *gridNP, double offsetX, double offsetY, double maxOffpix, double Scale, double Offset) { return true; }
+bool pmAstromVisualPlotTweak (psVector *xHist, psVector *yHist, int xBin, int yBin) {return true;}
+
+# endif
Index: branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.h
===================================================================
--- branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.h	(revision 20890)
+++ branches/bills_081204/psModules/src/astrom/pmAstrometryVisual.h	(revision 20890)
@@ -0,0 +1,6 @@
+bool pmAstromSetVisual(bool mode);
+bool pmAstromVisualInitWindow (int *kapid, char *name);
+bool pmAstromVisualAskUser (bool *plotflag);
+bool pmAstromVisualClose();
+bool pmAstromVisualPlotGridMatch (const psArray *raw, const psArray *ref, psImage *gridNP, double offsetX, double offsetY, double maxOffpix, double Scale, double Offset);
+bool pmAstromVisualPlotTweak (psVector *xHist, psVector *yHist, int xBin, int yBin);
Index: branches/bills_081204/psModules/src/astrom/pmAstrometryWCS.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/astrom/pmAstrometryWCS.c	(revision 20530)
+++ branches/bills_081204/psModules/src/astrom/pmAstrometryWCS.c	(revision 20890)
@@ -7,6 +7,6 @@
  *  @author EAM, IfA
  *
- *  @version $Revision: 1.29 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-09-12 01:05:59 $
+ *  @version $Revision: 1.31 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-13 19:38:16 $
  *
  *  Copyright 2006 Institute for Astronomy, University of Hawaii
@@ -272,4 +272,12 @@
     wcs->crpix2 = psMetadataLookupF64 (&status, header, "CRPIX2");
     wcs->toSky = psProjectionAlloc (wcs->crval1*PM_RAD_DEG, wcs->crval2*PM_RAD_DEG, PM_RAD_DEG, PM_RAD_DEG, type);
+
+    // These aren't needed but having them empty is disconcerting
+    strncpy(wcs->ctype2, ctype, PM_ASTROM_WCS_TYPE_SIZE-1);
+    ctype = psMetadataLookupStr (&status, header, "CTYPE1");
+    strncpy(wcs->ctype1, ctype, PM_ASTROM_WCS_TYPE_SIZE-1);
+    wcs->ctype1[PM_ASTROM_WCS_TYPE_SIZE-1] = 0;
+    wcs->ctype2[PM_ASTROM_WCS_TYPE_SIZE-1] = 0;
+
     // XXX I think this is wrong for linear proj
 
@@ -394,4 +402,12 @@
     }
 
+    // remove any existing 'CDi_j style' wcs keywords
+    if (psMetadataLookup(header, "CD1_1")) {
+        psMetadataRemoveKey(header, "CD1_1");
+        psMetadataRemoveKey(header, "CD1_2");
+        psMetadataRemoveKey(header, "CD2_1");
+        psMetadataRemoveKey(header, "CD2_2");
+    }
+
     return true;
 }
Index: branches/bills_081204/psModules/src/camera/pmFPACopy.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPACopy.c	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPACopy.c	(revision 20890)
@@ -172,4 +172,6 @@
     int xParityTarget = psMetadataLookupS32(&mdokT, target->concepts, "CELL.XPARITY"); // Target x parity
     assert(mdokS && mdokT);
+    psAssert (abs(xParitySource) == 1, "CELL.XPARITY not set for source");
+    psAssert (abs(xParityTarget) == 1, "CELL.XPARITY not set for target");
     if (abs(xParitySource) != 1) {
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "CELL.XPARITY is not set for source (%d)",
@@ -189,4 +191,6 @@
     int yParitySource = psMetadataLookupS32(&mdokS, source->concepts, "CELL.YPARITY"); // Source parity
     assert(mdokS && mdokT);
+    psAssert (abs(yParitySource) == 1, "CELL.YPARITY not set for source");
+    psAssert (abs(yParityTarget) == 1, "CELL.YPARITY not set for target");
     if (abs(yParitySource) != 1) {
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "CELL.YPARITY is not set for source (%d)",
@@ -324,4 +328,6 @@
         int xSize = psMetadataLookupS32(NULL, source->concepts, "CELL.XSIZE"); // CELL.XSIZE of source
 
+	psAssert (abs(xParity) == 1, "CELL.XPARITY not set for source");
+
         if (sourceBin == 0) {
             // Don't know the binning; assume it is unity
@@ -472,6 +478,8 @@
         pmCell *targetCell = pmCellAlloc (targetChip, cellName);
         int xParityTarget = psMetadataLookupS32(&status, sourceCell->concepts, "CELL.XPARITY"); // Target x parity
+	psAssert (abs(xParityTarget) == 1, "CELL.XPARITY not set for target");
         psMetadataAddS32 (targetCell->concepts, PS_LIST_TAIL, "CELL.XPARITY", PS_META_REPLACE, "", xParityTarget);
         int yParityTarget = psMetadataLookupS32(&status, sourceCell->concepts, "CELL.YPARITY"); // Target y parity
+	psAssert (abs(xParityTarget) == 1, "CELL.YPARITY not set for target");
         psMetadataAddS32 (targetCell->concepts, PS_LIST_TAIL, "CELL.YPARITY", PS_META_REPLACE, "", yParityTarget);
         if (!cellCopy(targetCell, sourceCell, true, 1, 1)) {
Index: branches/bills_081204/psModules/src/camera/pmFPAExtent.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPAExtent.c	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPAExtent.c	(revision 20890)
@@ -92,8 +92,26 @@
     }
 
-    cellExtent->x0 += x0;
-    cellExtent->x1 += x0;
-    cellExtent->y0 += y0;
-    cellExtent->y1 += y0;
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // CELL.X0,Y0 are the coordinate of the amp on the chip, subtract size if parity flipped
+    if (xParityCell > 0) {
+	cellExtent->x0 += x0;
+	cellExtent->x1 += x0;
+    } else {
+	float x0Cell = x0 - cellExtent->x1;
+	float x1Cell = x0 - cellExtent->x0;
+	cellExtent->x0 = x0Cell;
+	cellExtent->x1 = x1Cell;
+    }
+    if (yParityCell > 0) {
+	cellExtent->y0 += y0;
+	cellExtent->y1 += y0;
+    } else {
+	float y0Cell = y0 - cellExtent->y1;
+	float y1Cell = y0 - cellExtent->y0;
+	cellExtent->y0 = y0Cell;
+	cellExtent->y1 = y1Cell;
+    }
 
     return cellExtent;
Index: branches/bills_081204/psModules/src/camera/pmFPAMaskWeight.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPAMaskWeight.c	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPAMaskWeight.c	(revision 20890)
@@ -491,8 +491,8 @@
     for (int i = 0; i < num; i++) {
         float measuredSig = PS_SQR(source->data.F32[i] / stdev); // Measured significance
-        if (source->data.F32[i] <= 0.0) {
+        ratio->data.F32[i] = measuredSig / guess->data.F32[i];
+        if (guess->data.F32[i] <= 0.0 || source->data.F32[i] <= 0.0 || !isfinite(ratio->data.F32[i])) {
             photMask->data.PS_TYPE_MASK_DATA[i] = 0xFF;
         }
-        ratio->data.F32[i] = guess->data.F32[i] / measuredSig;
         psTrace("psModules.camera", 9, "Ratio %d: %f, %f, %f\n",
                 i, guess->data.F32[i], measuredSig, ratio->data.F32[i]);
Index: branches/bills_081204/psModules/src/camera/pmFPAfileDefine.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPAfileDefine.c	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPAfileDefine.c	(revision 20890)
@@ -77,5 +77,5 @@
 
 // define an input-type pmFPAfile, bind to the optional fpa if supplied
-pmFPAfile *pmFPAfileDefineInput(const pmConfig *config, pmFPA *fpa, const char *name)
+pmFPAfile *pmFPAfileDefineInput(const pmConfig *config, pmFPA *fpa, char *cameraName, const char *name)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -125,5 +125,5 @@
         file->fpa = psMemIncrRefCounter(fpa);
         file->camera = psMemIncrRefCounter((psMetadata *)fpa->camera);
-        file->cameraName = psMemIncrRefCounter(config->cameraName); // XXX Is this the correct thing to do?
+        file->cameraName = cameraName ? psMemIncrRefCounter(cameraName) : psMemIncrRefCounter(config->cameraName); // XXX Is this the correct thing to do?
     } else {
         file->camera = psMemIncrRefCounter(config->camera);
@@ -483,5 +483,6 @@
     // Determine the current format from the header; Determine camera if not specified already.
     // the returned pointers 'camera' and 'formatName' are allocated here
-    format = pmConfigCameraFormatFromHeader(&camera, &formatName, config, phu, true);
+    psString cameraName = NULL;
+    format = pmConfigCameraFormatFromHeader(&camera, &cameraName, &formatName, config, phu, true);
     if (!format) {
         psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
@@ -495,5 +496,5 @@
     // build the template fpa, set up the basic view
     // XXX do we want this to be the baseCamera name or the metaCamera name?
-    fpa = pmFPAConstruct(camera, config->cameraName);
+    fpa = pmFPAConstruct(camera, cameraName);
     if (!fpa) {
         psError(PS_ERR_IO, false, "Failed to construct FPA from %s", realName);
@@ -510,5 +511,5 @@
     // load the given filerule (from config->camera) and bind it to the fpa
     // the returned file is just a view to the entry on config->files
-    file = pmFPAfileDefineInput(config, fpa, filename);
+    file = pmFPAfileDefineInput(config, fpa, cameraName, filename);
     if (!file) {
         psError(PS_ERR_IO, false, "file %s not defined\n", filename);
@@ -519,5 +520,7 @@
         return NULL;
     }
+    psFree (cameraName);
     psFree (file->filerule); // this is set in pmFPAfileDefineInput
+
     file->format = format;
     file->formatName = formatName;
@@ -659,5 +662,5 @@
     // load the given filerule (from config->camera) and bind it to the fpa
     // the returned file is just a view to the entry on config->files
-    file = pmFPAfileDefineInput (config, input->fpa, filename);
+    file = pmFPAfileDefineInput (config, input->fpa, NULL, filename);
     if (!file) {
         psError(PS_ERR_IO, false, "file %s not defined\n", filename);
@@ -703,5 +706,5 @@
 
         if (!format) {
-            format = pmConfigCameraFormatFromHeader(NULL, NULL, config, phu, true);
+            format = pmConfigCameraFormatFromHeader(NULL, NULL, NULL, config, phu, true);
             if (!format) {
                 psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
@@ -816,5 +819,6 @@
     psMetadata *camera = NULL;
     psString formatName = NULL;
-    format = pmConfigCameraFormatFromHeader (&camera, &formatName, config, phu, true);
+    psString cameraName = NULL;
+    format = pmConfigCameraFormatFromHeader (&camera, &cameraName, &formatName, config, phu, true);
     if (!format) {
         psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", (char *)infiles->data[0]);
@@ -826,5 +830,5 @@
 
     // build the template fpa, set up the basic view
-    fpa = pmFPAConstruct (camera, config->cameraName);
+    fpa = pmFPAConstruct (camera, cameraName);
     if (!fpa) {
         psError(PS_ERR_IO, false, "Failed to construct FPA from %s", (char *)infiles->data[0]);
@@ -840,5 +844,5 @@
     // the returned file is just a view to the entry on config->files
     // we need a variable name here... (but in filerule)
-    file = pmFPAfileDefineInput (config, fpa, filename);
+    file = pmFPAfileDefineInput (config, fpa, cameraName, filename);
     if (!file) {
         psError(PS_ERR_IO, false, "file %s not defined\n", filename);
@@ -848,5 +852,7 @@
         return NULL;
     }
+    psFree(cameraName);
     FPA_TEST_ASSERT (file);
+
     file->format = format;
     file->formatName = formatName;
@@ -922,5 +928,5 @@
     // load the given filerule (from config->camera) and bind it to the fpa
     // the returned file is just a view to the entry on config->files
-    pmFPAfile *file = pmFPAfileDefineInput(config, fpa, filename);
+    pmFPAfile *file = pmFPAfileDefineInput(config, fpa, NULL, filename);
     psFree (fpa);
     if (!file) {
@@ -1003,5 +1009,5 @@
         // load the given filerule (from config->camera) and bind it to the fpa
         // the returned file is just a view to the entry on config->files
-        file = pmFPAfileDefineInput (config, fpa, filename);
+        file = pmFPAfileDefineInput (config, fpa, NULL, filename);
         if (!file) {
             psError(PS_ERR_IO, false, "file %s not defined\n", filename);
Index: branches/bills_081204/psModules/src/camera/pmFPAfileDefine.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPAfileDefine.h	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPAfileDefine.h	(revision 20890)
@@ -4,6 +4,6 @@
  * @author EAM, IfA
  *
- * @version $Revision: 1.17 $ $Name: not supported by cvs2svn $
- * @date $Date: 2008-05-12 21:41:55 $
+ * @version $Revision: 1.18 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-11 00:03:46 $
  * Copyright 2004-2005 Institute for Astronomy, University of Hawaii
  */
@@ -20,5 +20,5 @@
 // reference count is held by the config->files metadata.  Multiple file rules of the same name are permitted
 // if multiple is true.
-pmFPAfile *pmFPAfileDefineInput (const pmConfig *config, pmFPA *fpa, const char *name);
+pmFPAfile *pmFPAfileDefineInput (const pmConfig *config, pmFPA *fpa, char *cameraName, const char *name);
 
 // load the pmFPAfile information from the camera configuration data
Index: branches/bills_081204/psModules/src/camera/pmFPAfileIO.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/camera/pmFPAfileIO.c	(revision 20530)
+++ branches/bills_081204/psModules/src/camera/pmFPAfileIO.c	(revision 20890)
@@ -783,5 +783,6 @@
 	  psMetadata *camera = NULL;
 	  psString formatName = NULL;
-          file->format = pmConfigCameraFormatFromHeader (&camera, &formatName, config, phu, true);
+	  psString cameraName = NULL;
+          file->format = pmConfigCameraFormatFromHeader (&camera, &cameraName, &formatName, config, phu, true);
           if (!file->format) {
             psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", file->filename);
@@ -800,4 +801,5 @@
 	  psFree(camera);
 	  psFree(formatName);
+	  psFree(cameraName);
 
 	  // XXX this is really dangerous...
@@ -894,5 +896,5 @@
     if (!file->format) {
         // determine the format (camera is already known); do not load the recipe
-        file->format = pmConfigCameraFormatFromHeader (NULL, &file->formatName, config, phu, false);
+        file->format = pmConfigCameraFormatFromHeader (NULL, NULL, &file->formatName, config, phu, false);
         if (!file->format) {
             psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", file->filename);
Index: branches/bills_081204/psModules/src/concepts/pmConceptsAverage.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/concepts/pmConceptsAverage.c	(revision 20530)
+++ branches/bills_081204/psModules/src/concepts/pmConceptsAverage.c	(revision 20890)
@@ -28,4 +28,5 @@
 }
 
+
 bool pmConceptsAverageFPAs(pmFPA *target, psList *sources)
 {
@@ -37,4 +38,8 @@
     psTimeType timeSys = 0;             // Time system
     char *filter     = NULL;            // Filter
+    char *filterId   = NULL;            // Filter (parsed, abstract name)
+    char *telescope  = NULL;            // Telescope of origin
+    char *instrument = NULL;            // Instrument name
+    char *detector   = NULL;            // Detector name
 
     int num = 0;                        // Number of FPAs
@@ -48,4 +53,11 @@
         num++;
 
+#define COMPARE_STR(NAME, VALUE) \
+    if (strcmp(VALUE, psMetadataLookupStr(NULL, fpa->concepts, NAME)) != 0) { \
+        psWarning("Differing %s in use: %s vs %s\n", \
+                  NAME, VALUE, psMetadataLookupStr(NULL, fpa->concepts, NAME)); \
+        VALUE = "VARIOUS"; \
+    }
+
         psTime *fpaTime = psMetadataLookupPtr(NULL, fpa->concepts, "FPA.TIME");
         time       += psTimeToMJD(fpaTime);
@@ -53,4 +65,8 @@
             timeSys = psMetadataLookupS32(NULL, fpa->concepts, "FPA.TIMESYS");
             filter = psMetadataLookupStr(NULL, fpa->concepts, "FPA.FILTER");
+            filterId = psMetadataLookupStr(NULL, fpa->concepts, "FPA.FILTERID");
+            telescope = psMetadataLookupStr(NULL, fpa->concepts, "FPA.TELESCOPE");
+            instrument = psMetadataLookupStr(NULL, fpa->concepts, "FPA.INSTRUMENT");
+            detector = psMetadataLookupStr(NULL, fpa->concepts, "FPA.DETECTOR");
         } else {
             if (timeSys != psMetadataLookupS32(NULL, fpa->concepts, "FPA.TIMESYS")) {
@@ -58,16 +74,21 @@
                           timeSys, psMetadataLookupS32(NULL, fpa->concepts, "FPA.TIMESYS"));
             }
-            if (strcmp(filter, psMetadataLookupStr(NULL, fpa->concepts, "FPA.FILTER")) != 0) {
-                psWarning("Differing FPA.FILTER in use: %s vs %s\n",
-                          filter, psMetadataLookupStr(NULL, fpa->concepts, "FPA.FILTER"));
-            }
+            COMPARE_STR("FPA.FILTER", filter);
+            COMPARE_STR("FPA.FILTERID", filterId);
+            COMPARE_STR("FPA.TELESCOPE", telescope);
+            COMPARE_STR("FPA.INSTRUMENT", instrument);
+            COMPARE_STR("FPA.DETECTOR", detector);
         }
     }
     psFree(sourcesIter);
 
-    time      /= (double)num;
+    time /= (double)num;
 
     MD_UPDATE(target->concepts, "FPA.TIMESYS", S32, timeSys);
     MD_UPDATE_STR(target->concepts, "FPA.FILTER", filter);
+    MD_UPDATE_STR(target->concepts, "FPA.FILTERID", filterId);
+    MD_UPDATE_STR(target->concepts, "FPA.TELESCOPE", telescope);
+    MD_UPDATE_STR(target->concepts, "FPA.INSTRUMENT", instrument);
+    MD_UPDATE_STR(target->concepts, "FPA.DETECTOR", detector);
 
     // FPA.TIME needs special care
@@ -81,4 +102,29 @@
 }
 
+float averageWithDropouts (psList *sources, char *name) {
+
+    bool status;
+
+    float sum = 0;
+    int nCells = 0;                     // Number of cells;
+    psListIterator *sourcesIter = psListIteratorAlloc(sources, PS_LIST_HEAD, false); // Iterator for sources
+    pmCell *cell = NULL;                // Source cell from iteration
+    while ((cell = psListGetAndIncrement(sourcesIter))) {
+        if (!cell) {
+            continue;
+        }
+
+	float value = psMetadataLookupF32(&status, cell->concepts, name);
+	if (!status) continue;
+	if (!isfinite(value)) continue;
+
+        sum += value;
+        nCells++;
+    }
+    psFree (sourcesIter);
+
+    float average = sum / nCells;
+    return average;
+}
 
 // Set a variety of concepts in a cell by averaging over several
@@ -90,10 +136,6 @@
     PS_ASSERT_INT_POSITIVE(sources->n, false);
 
-    float gain       = 0.0;             // Gain
-    float readnoise  = 0.0;             // Read noise
     float saturation = INFINITY;        // Saturation level
     float bad        = -INFINITY;       // Bad level
-    float exposure   = 0.0;             // Exposure time
-    float darktime   = 0.0;             // Dark time
     double time      = 0.0;             // Time of observation
     psTimeType timeSys = 0;             // Time system
@@ -103,4 +145,10 @@
     int xParity = 0, yParity = 0;       // Parity
 
+    float gain      = averageWithDropouts (sources, "CELL.GAIN");
+    float readnoise = averageWithDropouts (sources, "CELL.READNOISE");
+    float exposure  = averageWithDropouts (sources, "CELL.EXPOSURE");
+    float darktime  = averageWithDropouts (sources, "CELL.DARKTIME");
+
+    // other concepts are a bit more "special"
     int nCells = 0;                     // Number of cells;
     psListIterator *sourcesIter = psListIteratorAlloc(sources, PS_LIST_HEAD, false); // Iterator for sources
@@ -112,8 +160,4 @@
 
         nCells++;
-        gain       += psMetadataLookupF32(NULL, cell->concepts, "CELL.GAIN");
-        readnoise  += psMetadataLookupF32(NULL, cell->concepts, "CELL.READNOISE");
-        exposure   += psMetadataLookupF32(NULL, cell->concepts, "CELL.EXPOSURE");
-        darktime   += psMetadataLookupF32(NULL, cell->concepts, "CELL.DARKTIME");
         psTime *cellTime = psMetadataLookupPtr(NULL, cell->concepts, "CELL.TIME");
         time       += psTimeToMJD(cellTime);
@@ -179,9 +223,5 @@
     psFree(sourcesIter);
 
-    gain      /= (float)nCells;
-    readnoise /= (float)nCells;
-    exposure  /= (float)nCells;
-    darktime  /= (float)nCells;
-    time      /= (double)nCells;
+    time /= (double) nCells;
 
     MD_UPDATE(target->concepts, "CELL.GAIN", F32, gain);
Index: branches/bills_081204/psModules/src/concepts/pmConceptsRead.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/concepts/pmConceptsRead.c	(revision 20530)
+++ branches/bills_081204/psModules/src/concepts/pmConceptsRead.c	(revision 20890)
@@ -72,4 +72,9 @@
     }
 
+    psTrace ("psModules.concepts", 3, "parsing concept: %s\n", spec->blank->name);
+    if (!strcmp (spec->blank->name, "CELL.XPARITY")) {
+        psTrace ("psModules.concepts", 3, "parsing CELL.XPARITY: %s\n", spec->blank->name);
+    }
+
     psMetadataItem *parsed = NULL;  // The parsed concept
     if (spec->parse) {
@@ -88,6 +93,4 @@
         }
     }
-
-    psTrace ("psModules.concepts", 3, "parsing concept: %s\n", spec->blank->name);
 
     // Plug the parsed concept into a new psMetadataItem, so each "concept" has its own version that can
@@ -270,4 +273,12 @@
         psMetadataItem *headerItem = NULL; // The value of the concept from the header
 
+        psTrace ("psModules.concepts", 3, "reading concept: %s\n", name);
+        if (!strcmp (name, "CELL.XPARITY")) {
+            psTrace ("psModules.concepts", 3, "parsing CELL.XPARITY: %s\n", name);
+        }
+        if (!strcmp (name, "CELL.TRIMSEC")) {
+            psTrace ("psModules.concepts", 3, "parsing CELL.TRIMSEC: %s\n", name);
+        }
+
         // First check the cell configuration
         if (cell && cell->config) {
Index: branches/bills_081204/psModules/src/concepts/pmConceptsStandard.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/concepts/pmConceptsStandard.c	(revision 20530)
+++ branches/bills_081204/psModules/src/concepts/pmConceptsStandard.c	(revision 20890)
@@ -113,5 +113,6 @@
                         float gain = psMetadataLookupF32(NULL, cell->concepts, "CELL.GAIN"); // Gain (e/ADU)
                         if (!isfinite(gain)) {
-                            psWarning("CELL.READNOISE is supposed to be in ADU, but CELL.GAIN isn't set");
+                            psWarning("CELL.READNOISE is supposed to be in ADU, but CELL.GAIN isn't set -- forcing to 1.0");
+			    gain = 1.0;
                         }
                         rn /= gain;
Index: branches/bills_081204/psModules/src/config/pmConfig.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/config/pmConfig.c	(revision 20530)
+++ branches/bills_081204/psModules/src/config/pmConfig.c	(revision 20890)
@@ -1081,6 +1081,6 @@
 // camera.  If we are discovering the camera (config->camera == NULL), then we also load the
 // recipe files for the camera.
-psMetadata *pmConfigCameraFormatFromHeader(psMetadata **camera, psString *formatName, pmConfig *config,
-                                           const psMetadata *header, bool readRecipes)
+psMetadata *pmConfigCameraFormatFromHeader(psMetadata **camera, psString *cameraName, psString *formatName, 
+					   pmConfig *config, const psMetadata *header, bool readRecipes)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -1089,5 +1089,5 @@
     bool status = false;                // error status
     psMetadata *format = NULL;          // The winning format
-    psString name = NULL;               // Name of the winning format
+    psString testFormatName = NULL;         // Name of the winning format
 
     // If we don't know what sort of camera we have, we try all that we know
@@ -1118,8 +1118,8 @@
             assert(camerasItem->type == PS_DATA_METADATA); // It should be because we've read it in or deleted
             psMetadata *testCamera = camerasItem->data.md; // Camera to test against what we've got:
-            if (formatFromHeader(&status, &format, &name, testCamera, header, camerasItem->name)) {
+            if (formatFromHeader(&status, &format, &testFormatName, testCamera, header, camerasItem->name)) {
                 config->camera = psMemIncrRefCounter(testCamera);
                 config->cameraName = psStringCopy(camerasItem->name);
-                config->formatName = name;
+                config->formatName = testFormatName;
                 config->format = format;
                 if (camera) {
@@ -1127,5 +1127,8 @@
                 }
                 if (formatName) {
-                    *formatName = psMemIncrRefCounter(name);    // view on value saved on config
+                    *formatName = psMemIncrRefCounter(testFormatName);    // view on value saved on config
+                }
+                if (cameraName) {
+                    *cameraName = psMemIncrRefCounter(config->cameraName);    // view on value saved on config
                 }
             } else {
@@ -1166,5 +1169,5 @@
 
     psMetadata *testCamera = NULL;
-    char *testName = NULL;
+    char *testCameraName = NULL;
 
     psMetadata *cameras = psMetadataLookupMetadata (NULL, config->system, "CAMERAS");
@@ -1173,50 +1176,50 @@
     // try the FPA metaCamera
     if (!found) {
-        testName = NULL;
-        psStringAppend (&testName, "_%s-FPA", baseName);
-
-        testCamera = psMetadataLookupMetadata (NULL, cameras, testName);
-        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testName);
+        testCameraName = NULL;
+        psStringAppend (&testCameraName, "_%s-FPA", baseName);
+
+        testCamera = psMetadataLookupMetadata (NULL, cameras, testCameraName);
+        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testCameraName);
 
         bool status;
-        found = formatFromHeader(&status, &format, &name, testCamera, header, testName);
-        if (!found) psFree (testName);
+        found = formatFromHeader(&status, &format, &testFormatName, testCamera, header, testCameraName);
+        if (!found) psFree (testCameraName);
     }
 
     // try the CHIP metaCamera
     if (!found) {
-        testName = NULL;
-        psStringAppend (&testName, "_%s-CHIP", baseName);
-
-        testCamera = psMetadataLookupMetadata (NULL, cameras, testName);
-        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testName);
+        testCameraName = NULL;
+        psStringAppend (&testCameraName, "_%s-CHIP", baseName);
+
+        testCamera = psMetadataLookupMetadata (NULL, cameras, testCameraName);
+        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testCameraName);
 
         bool status;
-        found = formatFromHeader(&status, &format, &name, testCamera, header, testName);
-        if (!found) psFree (testName);
+        found = formatFromHeader(&status, &format, &testFormatName, testCamera, header, testCameraName);
+        if (!found) psFree (testCameraName);
     }
 
     // try the SKYCELL metaCamera
     if (!found) {
-        testName = NULL;
-        psStringAppend (&testName, "_%s-SKYCELL", baseName);
-
-        testCamera = psMetadataLookupMetadata (NULL, cameras, testName);
-        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testName);
+        testCameraName = NULL;
+        psStringAppend (&testCameraName, "_%s-SKYCELL", baseName);
+
+        testCamera = psMetadataLookupMetadata (NULL, cameras, testCameraName);
+        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testCameraName);
 
         bool status;
-        found = formatFromHeader(&status, &format, &name, testCamera, header, testName);
-        if (!found) psFree (testName);
+        found = formatFromHeader(&status, &format, &testFormatName, testCamera, header, testCameraName);
+        if (!found) psFree (testCameraName);
     }
 
     // try the base name
     if (!found) {
-        testName = psMemIncrRefCounter (baseName);
-
-        testCamera = psMetadataLookupMetadata (NULL, cameras, testName);
-        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testName);
+        testCameraName = psMemIncrRefCounter (baseName);
+
+        testCamera = psMetadataLookupMetadata (NULL, cameras, testCameraName);
+        psAssert (testCamera, "missing %s in CAMERAS in complete metadata", testCameraName);
 
         bool status;
-        found = formatFromHeader(&status, &format, &name, testCamera, header, testName);
+        found = formatFromHeader(&status, &format, &testFormatName, testCamera, header, testCameraName);
     }
 
@@ -1228,12 +1231,16 @@
     }
 
-    psFree (name); // winning format name (for metaCamera) returned by formatFromHeader
-
     psFree (baseName);
     if (formatName) {
-        *formatName = testName;
+        *formatName = testFormatName;
     } else {
-        psFree (testName);
-    }
+        psFree (testFormatName);
+    }
+    if (cameraName) {
+        *cameraName = testCameraName;
+    } else {
+        psFree (testCameraName);
+    }
+
     if (camera) {
         *camera = psMemIncrRefCounter(testCamera);
@@ -1359,5 +1366,5 @@
         if (hdrItem && psMetadataItemCompare(hdrItem, rulesItem)) {
             // It's already there and matches
-            break;
+            continue;
         }
 
Index: branches/bills_081204/psModules/src/config/pmConfig.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/config/pmConfig.h	(revision 20530)
+++ branches/bills_081204/psModules/src/config/pmConfig.h	(revision 20890)
@@ -5,6 +5,6 @@
  *  @author Eugene Magnier, IfA
  *
- *  @version $Revision: 1.41 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-08-06 01:13:14 $
+ *  @version $Revision: 1.42 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-11 00:04:47 $
  *  Copyright 2005-2006 Institute for Astronomy, University of Hawaii
  */
@@ -117,4 +117,5 @@
 psMetadata *pmConfigCameraFormatFromHeader(psMetadata **camera, // selected camera (or meta-camera)
                                            psString *formatName, // selected format name
+                                           psString *cameraName, // selected camera name
                                            pmConfig *config, ///< The configuration
                                            const psMetadata *header, ///< The FITS header
Index: branches/bills_081204/psModules/src/detrend/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/psModules/src/detrend/Makefile.am	(revision 20530)
+++ branches/bills_081204/psModules/src/detrend/Makefile.am	(revision 20890)
@@ -15,5 +15,6 @@
 	pmDetrendThreads.c \
 	pmShifts.c \
-	pmDark.c
+	pmDark.c \
+	pmRemnance.c
 
 #	pmSkySubtract.c
@@ -31,5 +32,6 @@
 	pmDetrendThreads.h \
 	pmShifts.h \
-	pmDark.h
+	pmDark.h \
+	pmRemnance.h
 
 #	pmSkySubtract.h
Index: branches/bills_081204/psModules/src/detrend/pmRemnance.c
===================================================================
--- branches/bills_081204/psModules/src/detrend/pmRemnance.c	(revision 20890)
+++ branches/bills_081204/psModules/src/detrend/pmRemnance.c	(revision 20890)
@@ -0,0 +1,100 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+
+#include "pmFPA.h"
+#include "pmRemnance.h"
+
+#define SIZE 30                         // Size of accumulation patch
+#define THRESHOLD 20.0                   // Threshold above background
+
+bool pmRemnance(pmReadout *ro,           ///< Readout with input image
+                psMaskType maskVal,      ///< Value of mask
+                psMaskType maskRem,       ///< Value to give remance
+                int size,               ///< Size of accumulation patches
+                float threshold         ///< Threshold for masking
+    )
+{
+    PM_ASSERT_READOUT_NON_NULL(ro, false);
+    PM_ASSERT_READOUT_IMAGE(ro, false);
+    PM_ASSERT_READOUT_MASK(ro, false);
+
+    psImage *image = ro->image, *mask = ro->mask; // Mask and image from readout
+    int numCols = image->numCols, numRows = image->numRows; // Size of image
+
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
+
+    psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics
+    if (!psImageBackground(stats, NULL, image, mask, maskVal, rng)) {
+        // Probably means the entire readout is bad
+        psErrorClear();
+        psWarning("Unable to calculate image statistics: masking entire readout.");
+        psBinaryOp(mask, mask, "|", psScalarAlloc(maskRem, PS_TYPE_MASK));
+        psFree(stats);
+        psFree(rng);
+        return true;
+    }
+    psFree(rng);
+    float bgMean = stats->robustMedian; // Background level
+    float bgStdev = stats->robustStdev; // Background stdev
+
+    stats->options = PS_STAT_SAMPLE_MEDIAN;
+
+    int numMasked = 0;                  // Number of pixels masked
+    int number = ceil(numRows / (float)SIZE); // Number of steps up the columns
+    psVector *values = psVectorAlloc(numRows, PS_TYPE_F32); // Values below center
+    for (int x = 0; x < numCols; x++) {
+        int maxMask = 0;                // Maximum row to which to mask
+        int numValues = 0;              // Number of values
+        bool done = false;              // Are we done yet?
+        for (int i = 0, min = 0, max = size; i < number && !done; i++, min += size, max += size) {
+
+            if (max > numRows) {
+                max = numRows;
+            }
+            for (int y = min; y < max; y++) {
+                if (mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal) {
+                    continue;
+                }
+                values->data.F32[numValues++] = image->data.F32[y][x];
+            }
+            values->n = numValues;
+            if (!psVectorStats(stats, values, NULL, NULL, 0)) {
+                // Can't do anything about it
+                psErrorClear();
+                maxMask = max;
+                continue;
+            }
+            float median = stats->sampleMedian;
+
+            if (median > bgMean + threshold * bgStdev / sqrtf(numValues)) {
+                maxMask = max;
+            } else {
+                done = true;
+            }
+        }
+
+        if (maxMask > 0) {
+            maxMask += size;
+            if (maxMask > numRows) {
+                maxMask = numRows;
+            }
+            for (int y = 0; y < maxMask; y++) {
+                mask->data.PS_TYPE_MASK_DATA[y][x] |= maskRem;
+            }
+            numMasked += maxMask;
+        }
+
+    }
+    psFree(values);
+    psFree(stats);
+
+
+    psMetadataAddS32(ro->analysis, PS_LIST_TAIL, PM_REMNANCE_ANALYSIS_NUM, 0,
+                     "Number of remnance pixels masked", numMasked);
+
+    return true;
+}
Index: branches/bills_081204/psModules/src/detrend/pmRemnance.h
===================================================================
--- branches/bills_081204/psModules/src/detrend/pmRemnance.h	(revision 20890)
+++ branches/bills_081204/psModules/src/detrend/pmRemnance.h	(revision 20890)
@@ -0,0 +1,21 @@
+#ifndef PM_REMNANCE_H
+#define PM_REMNANCE_H
+
+#include <pslib.h>
+#include "pmFPA.h"
+
+// Analysis metadata names
+#define PM_REMNANCE_ANALYSIS_NUM "DETREND.REMNANCE.NUM" // Number of masked remnance pixels
+
+// Mask remnance pixels
+//
+// By "remnance", we mean pixels left over from previous exposures.
+// GPC1 leaves remnance that flows down from where the annoyed pixels are.
+bool pmRemnance(pmReadout *ro,           ///< Readout with input image
+                psMaskType maskVal,      ///< Value of mask
+                psMaskType maskRem,       ///< Value to give remance
+                int size,               ///< Size of accumulation patches
+                float threshold         ///< Threshold for masking
+    );
+
+#endif
Index: branches/bills_081204/psModules/src/imcombine/pmStack.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmStack.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmStack.c	(revision 20890)
@@ -8,6 +8,6 @@
  *  @author GLG, MHPCC
  *
- *  @version $Revision: 1.44 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-11-01 02:59:33 $
+ *  @version $Revision: 1.45 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-13 03:50:30 $
  *  Copyright 2004-2007 Institute for Astronomy, University of Hawaii
  *
@@ -353,11 +353,11 @@
                   float rej2 = PS_SQR(rej); // Rejection level squared
                   for (int i = 0; i < num; i++) {
-                      pixelVariances->data.F32[i] *= rej2;
+                      // Systematic error contributes to the rejection level
+                      pixelVariances->data.F32[i] += PS_SQR(sys * pixelData->data.F32[i]);
 #ifdef VARIANCE_FACTORS
                       // Variance factor contributes to the rejection level
                       pixelVariances->data.F32[i] *= varFactors->data.F32[pixelSources->data.U16[i]];
 #endif
-                      // Systematic error contributes to the rejection level
-                      pixelVariances->data.F32[i] += PS_SQR(sys * pixelData->data.F32[i]);
+                      pixelVariances->data.F32[i] *= rej2;
                   }
               }
Index: branches/bills_081204/psModules/src/imcombine/pmStackReject.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmStackReject.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmStackReject.c	(revision 20890)
@@ -126,5 +126,5 @@
 
 psPixels *pmStackReject(const psPixels *in, int numCols, int numRows, float threshold, float poorFrac,
-                        const psArray *subRegions, const psArray *subKernels)
+                        int stride, const psArray *subRegions, const psArray *subKernels)
 {
     PS_ASSERT_PIXELS_NON_NULL(in, NULL);
@@ -168,5 +168,5 @@
         psRegion *region = subRegions->data[i]; // Region of interest
         pmSubtractionKernels *kernels = subKernels->data[i]; // Kernel of interest
-        if (!pmSubtractionConvolve(convRO, NULL, inRO, NULL, NULL, 0, 0, 1.0, NAN,
+        if (!pmSubtractionConvolve(convRO, NULL, inRO, NULL, NULL, stride, 0, 0, 1.0, 0.0,
                                    region, kernels, false, true)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to convolve mask image in region %d.", i);
@@ -245,4 +245,21 @@
     psImage *source = psPixelsToMask(NULL, bad, psRegionSet(0, numCols - 1, 0, numRows - 1),
                                      PM_STACK_MASK_BAD); // Mask image to grow
+
+#ifdef TESTING
+    {
+        static int seqNum = 0;          // Sequence number
+        psString name = NULL;           // Name of image
+        psStringAppend(&name, "reject_orig_%02d.fits", seqNum);
+        seqNum++;
+        psFits *fits = psFitsOpen(name, "w"); // FITS file pointer
+        psFree(name);
+        psFitsWriteImage(fits, NULL, source, 0, NULL);
+        psFitsClose(fits);
+    }
+#endif
+
+    // Need to set psImageConvolveMask threading OFF because that would generate threads on top of threads
+    bool oldThreads = psImageConvolveSetThreads(false); // Old value of threading for psImageColvolve
+
     psImage *target = psImageRecycle(convolved, numCols, numRows, PS_TYPE_MASK); // Grown image
     psImageInit(target, 0);
@@ -310,4 +327,6 @@
         psThreadJob *job;                   // Job to destroy
         while ((job = psThreadJobGetDone())) {
+            psAssert(strcmp(job->type, "PSMODULES_STACK_REJECT_GROW") == 0,
+                     "Job has incorrect type: %s", job->type);
             psFree(job);
         }
@@ -315,4 +334,19 @@
         psMutexDestroy(source);
     }
+
+    psImageConvolveSetThreads(oldThreads);
+
+#ifdef TESTING
+    {
+        static int seqNum = 0;          // Sequence number
+        psString name = NULL;           // Name of image
+        psStringAppend(&name, "reject_grow_%02d.fits", seqNum);
+        seqNum++;
+        psFits *fits = psFitsOpen(name, "w"); // FITS file pointer
+        psFree(name);
+        psFitsWriteImage(fits, NULL, target, 0, NULL);
+        psFitsClose(fits);
+    }
+#endif
 
     psFree(source);
Index: branches/bills_081204/psModules/src/imcombine/pmStackReject.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmStackReject.h	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmStackReject.h	(revision 20890)
@@ -13,4 +13,5 @@
                         float threshold, ///< Threshold on convolved image, 0..1
                         float poorFrac, ///< Fraction for "poor"
+                        int stride,     ///< Size of convolution patches
                         const psArray *regions, ///< Array of image regions for image
                         const psArray *kernels ///< Array of kernel parameters for each region
Index: branches/bills_081204/psModules/src/imcombine/pmSubtraction.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtraction.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtraction.c	(revision 20890)
@@ -1088,7 +1088,7 @@
 
 bool pmSubtractionConvolve(pmReadout *out1, pmReadout *out2, const pmReadout *ro1, const pmReadout *ro2,
-                           psImage *subMask, psMaskType maskBad, psMaskType maskPoor, float poorFrac,
-                           float sysError, const psRegion *region, const pmSubtractionKernels *kernels,
-                           bool doBG, bool useFFT)
+                           psImage *subMask, int stride, psMaskType maskBad, psMaskType maskPoor,
+                           float poorFrac, float sysError, const psRegion *region,
+                           const pmSubtractionKernels *kernels, bool doBG, bool useFFT)
 {
     int numCols = 0, numRows = 0;       // Image dimensions
@@ -1124,4 +1124,9 @@
         PS_ASSERT_IMAGE_SIZE(subMask, numCols, numRows, false);
     }
+    PS_ASSERT_INT_NONNEGATIVE(stride, false);
+    PS_ASSERT_FLOAT_LARGER_THAN_OR_EQUAL(poorFrac, 0.0, false);
+    PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(poorFrac, 1.0, false);
+    PS_ASSERT_FLOAT_LARGER_THAN_OR_EQUAL(sysError, 0.0, false);
+    PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(sysError, 1.0, false);
     if (region && psRegionIsNaN(*region)) {
         psString string = psRegionToString(*region);
@@ -1130,4 +1135,6 @@
         return false;
     }
+
+    psTimerStart("pmSubtractionConvolve");
 
     bool threaded = pmSubtractionThreaded(); // Running threaded?
@@ -1210,5 +1217,4 @@
 
     int size = kernels->size;           // Half-size of kernel
-    int fullSize = 2 * size + 1;        // Full size of kernel
 
     // Get region for convolution: [xMin:xMax,yMin:yMax]
@@ -1232,10 +1238,15 @@
     // and everything is executing psThreadPoolWait, waiting for some other mythical thread to complete the
     // thread's work.
-    psImageConvolveSetThreads(false);
-
-    for (int j = yMin; j < yMax; j += fullSize) {
-        int ySubMax = PS_MIN(j + fullSize, yMax); // Range for subregion of interest
-        for (int i = xMin; i < xMax; i += fullSize) {
-            int xSubMax = PS_MIN(i + fullSize, xMax); // Range for subregion of interest
+    bool oldThreads = psImageConvolveSetThreads(false); // Old value of threading for psImageConvolve
+
+    if (stride == 0) {
+        // Use the full size of the kernel
+        stride = 2 * size + 1;
+    }
+
+    for (int j = yMin; j < yMax; j += stride) {
+        int ySubMax = PS_MIN(j + stride, yMax); // Range for subregion of interest
+        for (int i = xMin; i < xMax; i += stride) {
+            int xSubMax = PS_MIN(i + stride, xMax); // Range for subregion of interest
 
             psRegion *subRegion = psRegionAlloc(i, xSubMax, j, ySubMax); // Bounds of subtraction
@@ -1308,4 +1319,6 @@
         psThreadJob *job;               // Completed job
         while ((job = psThreadJobGetDone())) {
+            psAssert(strcmp(job->type, "PSMODULES_SUBTRACTION_CONVOLVE") == 0,
+                     "Job has incorrect type: %s", job->type);
             psFree(job);
         }
@@ -1321,4 +1334,6 @@
         }
     }
+
+    psImageConvolveSetThreads(oldThreads);
 
     psFree(sys1);
@@ -1347,4 +1362,7 @@
     }
 
+    psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolve image: %f sec",
+             psTimerClear("pmSubtractionConvolve"));
+
     return true;
 }
Index: branches/bills_081204/psModules/src/imcombine/pmSubtraction.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtraction.h	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtraction.h	(revision 20890)
@@ -6,6 +6,6 @@
  * @author GLG, MHPCC
  *
- * @version $Revision: 1.33 $ $Name: not supported by cvs2svn $
- * @date $Date: 2008-11-01 03:36:01 $
+ * @version $Revision: 1.34 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-07 00:03:18 $
  * Copyright 2004-207 Institute for Astronomy, University of Hawaii
  */
@@ -103,4 +103,5 @@
                            const pmReadout *ro2, // Input image 2
                            psImage *subMask, ///< Subtraction mask (or NULL)
+                           int stride,  ///< Size of convolution patches
                            psMaskType maskBad, ///< Mask value to give bad pixels
                            psMaskType maskPoor, ///< Mask value to give poor pixels
Index: branches/bills_081204/psModules/src/imcombine/pmSubtractionEquation.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtractionEquation.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtractionEquation.c	(revision 20890)
@@ -665,4 +665,6 @@
     PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false);
 
+    psTimerStart("pmSubtractionCalculateEquation");
+
     // We iterate over each stamp, allocate the matrix and vectors if
     // necessary, and then calculate those matrix/vectors.
@@ -692,4 +694,8 @@
         return false;
     }
+
+    psLogMsg("psModules.imcombine", PS_LOG_INFO, "Calculate equation: %f sec",
+             psTimerClear("pmSubtractionCalculateEquation"));
+
 
     return true;
Index: branches/bills_081204/psModules/src/imcombine/pmSubtractionMask.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtractionMask.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtractionMask.c	(revision 20890)
@@ -137,4 +137,6 @@
     // footprint's distance of those (within 'footprint').
 
+    bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolve
+
     if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_1,
                              PM_SUBTRACTION_MASK_CONVOLVE_1,
@@ -151,4 +153,6 @@
         return NULL;
     }
+
+    psImageConvolveSetThreads(oldThreads);
 
     return mask;
Index: branches/bills_081204/psModules/src/imcombine/pmSubtractionMatch.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtractionMatch.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtractionMatch.c	(revision 20890)
@@ -92,5 +92,5 @@
 
 bool pmSubtractionMatch(pmReadout *conv1, pmReadout *conv2, const pmReadout *ro1, const pmReadout *ro2,
-                        int footprint, float regionSize, float stampSpacing, float threshold,
+                        int footprint, int stride, float regionSize, float stampSpacing, float threshold,
                         const psArray *sources, const char *stampsName,
                         pmSubtractionKernelsType type, int size, int spatialOrder,
@@ -139,4 +139,5 @@
 
     PS_ASSERT_INT_NONNEGATIVE(footprint, false);
+    PS_ASSERT_INT_NONNEGATIVE(stride, false);
     // regionSize can be just about anything (except maybe negative, but it can be NAN)
     PS_ASSERT_FLOAT_LARGER_THAN(stampSpacing, 0.0, false);
@@ -209,6 +210,9 @@
     pmSubtractionStampList *stamps = NULL; // Stamps for matching PSF
     pmSubtractionKernels *kernels = NULL; // Kernel basis functions
+    psMetadata *analysis = psMetadataAlloc(); // QA data
 
     int numCols = ro1->image->numCols, numRows = ro1->image->numRows; // Image dimensions
+
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
 
     memCheck("start");
@@ -218,6 +222,5 @@
     if (!subMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask.");
-        psFree(weight);
-        return false;
+        goto MATCH_ERROR;
     }
 
@@ -235,5 +238,15 @@
     }
 
-    psMetadata *analysis = psMetadataAlloc(); // QA data
+    {
+        psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for backgroun
+        if (!psImageBackground(bg, NULL, ro1->image, ro1->mask, maskVal, rng)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to measure background statistics.");
+            psFree(bg);
+            psFree(rng);
+            goto MATCH_ERROR;
+        }
+        threshold = bg->robustMedian + threshold * bg->robustStdev;
+        psFree(bg);
+    }
 
     // Iterate over iso-kernel regions
@@ -269,10 +282,8 @@
                 // Get backgrounds
                 psStats *bgStats = psStatsAlloc(BG_STAT); // Statistics for background
-                psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
                 psVector *buffer = NULL;// Buffer for stats
                 if (!psImageBackground(bgStats, &buffer, ro1->image, ro1->mask, maskVal, rng)) {
                     psError(PS_ERR_UNKNOWN, false, "Unable to measure background of image 1.");
                     psFree(bgStats);
-                    psFree(rng);
                     psFree(buffer);
                     goto MATCH_ERROR;
@@ -282,5 +293,4 @@
                     psError(PS_ERR_UNKNOWN, false, "Unable to measure background of image 2.");
                     psFree(bgStats);
-                    psFree(rng);
                     psFree(buffer);
                     goto MATCH_ERROR;
@@ -288,5 +298,4 @@
                 float bg2 = psStatsGetValue(bgStats, BG_STAT); // Background for image 2
                 psFree(bgStats);
-                psFree(rng);
                 psFree(buffer);
 
@@ -396,5 +405,5 @@
 
             psTrace("psModules.imcombine", 2, "Convolving...\n");
-            if (!pmSubtractionConvolve(conv1, conv2, ro1, ro2, subMask, maskBad, maskPoor, poorFrac,
+            if (!pmSubtractionConvolve(conv1, conv2, ro1, ro2, subMask, stride, maskBad, maskPoor, poorFrac,
                                        sysError, region, kernels, true, useFFT)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to convolve image.");
@@ -422,4 +431,6 @@
         }
     }
+    psFree(rng);
+    rng = NULL;
     psFree(region);
     region = NULL;
@@ -472,4 +483,5 @@
     psFree(stamps);
     psFree(weight);
+    psFree(rng);
     return false;
 }
Index: branches/bills_081204/psModules/src/imcombine/pmSubtractionMatch.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtractionMatch.h	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtractionMatch.h	(revision 20890)
@@ -16,4 +16,5 @@
                         // Stamp parameters
                         int footprint,  ///< Stamp half-size
+                        int stride,     ///< Size for convolution patches
                         float regionSize, ///< Typical size of iso-kernel regions
                         float stampSpacing, ///< Typical spacing between stamps
Index: branches/bills_081204/psModules/src/imcombine/pmSubtractionStamps.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/imcombine/pmSubtractionStamps.c	(revision 20530)
+++ branches/bills_081204/psModules/src/imcombine/pmSubtractionStamps.c	(revision 20890)
@@ -301,5 +301,5 @@
                     fluxList->n = index;
 
-                    goodStamp = true;
+                    goodStamp = (fluxStamp > threshold) ? true : false;
                 } else {
                     psTrace("psModules.imcombine", 9, "No sources in subregion %d", i);
Index: branches/bills_081204/psModules/src/objects/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/Makefile.am	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/Makefile.am	(revision 20890)
@@ -35,8 +35,10 @@
      pmSourceIO_PS1_DEV_0.c \
      pmSourceIO_PS1_DEV_1.c \
+     pmSourceIO_PS1_CAL_0.c \
      pmSourcePlots.c \
      pmSourcePlotPSFModel.c \
      pmSourcePlotMoments.c \
      pmSourcePlotApResid.c \
+     pmSourceVisual.c \
      pmResiduals.c \
      pmPSF.c \
@@ -74,4 +76,5 @@
      pmSourceIO.h \
      pmSourcePlots.h \
+     pmSourceVisual.h \
      pmResiduals.h \
      pmPSF.h \
Index: branches/bills_081204/psModules/src/objects/pmModel.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/pmModel.c	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/pmModel.c	(revision 20890)
@@ -6,6 +6,6 @@
  *  @author EAM, IfA
  *
- *  @version $Revision: 1.24 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-29 00:01:22 $
+ *  @version $Revision: 1.25 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-09 00:28:18 $
  *
  *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
@@ -211,6 +211,6 @@
 
     // apply optional relative offset
-    params->data.F32[PM_PAR_XPOS] += dx;
-    params->data.F32[PM_PAR_YPOS] += dy;
+    // params->data.F32[PM_PAR_XPOS] += dx;
+    // params->data.F32[PM_PAR_YPOS] += dy;
 
     // use these values for this realization
@@ -232,6 +232,6 @@
 
     if (model->residuals) {
-	DX = xBin*(image->col0 - xCenter) + model->residuals->xCenter + 0.5;
-	DY = yBin*(image->row0 - yCenter) + model->residuals->yCenter + 0.5;
+	DX = xBin*(image->col0 - xCenter - dx) + model->residuals->xCenter + 0.5;
+	DY = yBin*(image->row0 - yCenter - dy) + model->residuals->yCenter + 0.5;
 	Ro = (model->residuals->Ro)   ? model->residuals->Ro->data.F32 : NULL;
 	Rx = (model->residuals->Rx)   ? model->residuals->Rx->data.F32 : NULL;
@@ -254,6 +254,6 @@
             // XXX should we use using 0.5 pixel offset?
 	    // Convert to coordinate in parent image, with offset (dx,dy)
-            imageCol = ix + image->col0;
-            imageRow = iy + image->row0;
+            imageCol = ix + image->col0 - dx;
+            imageRow = iy + image->row0 - dy;
 
             x->data.F32[0] = (float) imageCol;
Index: branches/bills_081204/psModules/src/objects/pmPSFtry.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/pmPSFtry.c	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/pmPSFtry.c	(revision 20890)
@@ -5,6 +5,6 @@
  *  @author EAM, IfA
  *
- *  @version $Revision: 1.65 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-29 01:04:27 $
+ *  @version $Revision: 1.66 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-08 01:52:34 $
  *
  *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
@@ -34,4 +34,5 @@
 #include "pmSourceFitModel.h"
 #include "pmSourcePhotometry.h"
+#include "pmSourceVisual.h"
 
 bool printTrendMap (pmTrend2D *trend) {
@@ -683,4 +684,5 @@
 	    stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E0 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e0->n);
+	    pmSourceVisualPSFModelResid (trend, x, y, e0, srcMask);
 
 	    trend = psf->params->data[PM_PAR_E1];
@@ -689,4 +691,5 @@
 	    stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E1 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e1->n);
+	    pmSourceVisualPSFModelResid (trend, x, y, e1, srcMask);
 
 	    trend = psf->params->data[PM_PAR_E2];
@@ -695,4 +698,5 @@
 	    stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E2 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e2->n);
+	    pmSourceVisualPSFModelResid (trend, x, y, e2, srcMask);
 
             if (!status) {
@@ -853,4 +857,5 @@
 	psImageMapCleanup (trend->map);
 	// printTrendMap (trend);
+	pmSourceVisualPSFModelResid (trend, x, y, e0obs, mask);
 
 	trend = psf->params->data[PM_PAR_E1];
@@ -862,4 +867,5 @@
 	psImageMapCleanup (trend->map);
 	// printTrendMap (trend);
+	pmSourceVisualPSFModelResid (trend, x, y, e1obs, mask);
 
 	trend = psf->params->data[PM_PAR_E2];
@@ -871,4 +877,5 @@
 	psImageMapCleanup (trend->map);
 	// printTrendMap (trend);
+	pmSourceVisualPSFModelResid (trend, x, y, e2obs, mask);
     }
     psf->psfTrendStats->clipIter = nIter; // restore default setting
Index: branches/bills_081204/psModules/src/objects/pmSourceIO.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/pmSourceIO.c	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/pmSourceIO.c	(revision 20890)
@@ -3,6 +3,6 @@
  *  @author EAM, IfA
  *
- *  @version $Revision: 1.66 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2008-10-03 02:11:48 $
+ *  @version $Revision: 1.68 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-25 01:27:52 $
  *
  *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
@@ -440,8 +440,11 @@
                     }
                 }
-
                 if (numCols == 0 || numRows == 0) {
                     psWarning("Output source file has invalid IMNAXIS1, IMNAXIS2.");
                 }
+                psMetadataAddS32(hdu->header, PS_LIST_TAIL, "IMNAXIS1", PS_META_REPLACE,
+                                 "Number of columns in original image", numCols);
+                psMetadataAddS32(hdu->header, PS_LIST_TAIL, "IMNAXIS2", PS_META_REPLACE,
+                                 "Number of rows in original image", numRows);
 
                 psFitsWriteBlank (file->fits, hdu->header, headname);
@@ -485,12 +488,21 @@
                 status = pmSourcesWrite_PS1_DEV_1 (file->fits, sources, file->header, outhead, dataname, xsrcname);
             }
+            if (!strcmp (exttype, "PS1_CAL_0")) {
+                status = pmSourcesWrite_PS1_CAL_0 (file->fits, readout, sources, file->header, outhead, dataname, xsrcname);
+            }
             if (xsrcname) {
               if (!strcmp (exttype, "PS1_DEV_1")) {
-                status = pmSourcesWrite_PS1_DEV_1_XSRC (file->fits, sources, xsrcname, recipe);
+                  status = pmSourcesWrite_PS1_DEV_1_XSRC (file->fits, sources, xsrcname, recipe);
+              }
+              if (!strcmp (exttype, "PS1_CAL_0")) {
+                  status = pmSourcesWrite_PS1_CAL_0_XSRC (file->fits, readout, sources, file->header, xsrcname, recipe);
               }
             }
             if (xfitname) {
               if (!strcmp (exttype, "PS1_DEV_1")) {
-                status = pmSourcesWrite_PS1_DEV_1_XFIT (file->fits, sources, xfitname);
+                  status = pmSourcesWrite_PS1_DEV_1_XFIT (file->fits, sources, xfitname);
+              }
+              if (!strcmp (exttype, "PS1_CAL_0")) {
+                  status = pmSourcesWrite_PS1_CAL_0_XFIT (file->fits, readout, sources, file->header, xfitname);
               }
             }
@@ -915,4 +927,7 @@
                 sources = pmSourcesRead_PS1_DEV_1 (file->fits, hdu->header);
             }
+            if (!strcmp (exttype, "PS1_CAL_)")) {
+                sources = pmSourcesRead_PS1_CAL_0 (file->fits, hdu->header);
+            }
         }
 
Index: branches/bills_081204/psModules/src/objects/pmSourceIO.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/pmSourceIO.h	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/pmSourceIO.h	(revision 20890)
@@ -4,6 +4,6 @@
  * @author EAM, IfA; GLG, MHPCC
  *
- * @version $Revision: 1.18 $ $Name: not supported by cvs2svn $
- * @date $Date: 2008-07-17 22:38:15 $
+ * @version $Revision: 1.19 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-14 02:11:45 $
  * Copyright 2004 Maui High Performance Computing Center, University of Hawaii
  *
@@ -30,4 +30,8 @@
 bool pmSourcesWrite_PS1_DEV_1_XFIT (psFits *fits, psArray *sources, char *extname);
 
+bool pmSourcesWrite_PS1_CAL_0 (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname, char *xsrcname);
+bool pmSourcesWrite_PS1_CAL_0_XSRC (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe);
+bool pmSourcesWrite_PS1_CAL_0_XFIT (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname);
+
 bool pmSource_CMF_WritePHU (const pmFPAview *view, pmFPAfile *file, pmConfig *config);
 
@@ -37,4 +41,5 @@
 psArray *pmSourcesRead_PS1_DEV_0 (psFits *fits, psMetadata *header);
 psArray *pmSourcesRead_PS1_DEV_1 (psFits *fits, psMetadata *header);
+psArray *pmSourcesRead_PS1_CAL_0 (psFits *fits, psMetadata *header);
 
 bool pmSourcesWritePSFs (psArray *sources, char *filename);
Index: branches/bills_081204/psModules/src/objects/pmSourceIO_PS1_CAL_0.c
===================================================================
--- branches/bills_081204/psModules/src/objects/pmSourceIO_PS1_CAL_0.c	(revision 20890)
+++ branches/bills_081204/psModules/src/objects/pmSourceIO_PS1_CAL_0.c	(revision 20890)
@@ -0,0 +1,683 @@
+/** @file  pmSourceIO.c
+ *
+ *  @author EAM, IfA
+ *
+ *  @version $Revision: 1.1 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-11-14 02:11:45 $
+ *
+ *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <math.h>
+#include <string.h>
+#include <pslib.h>
+
+#include "pmConfig.h"
+#include "pmDetrendDB.h"
+
+#include "pmHDU.h"
+#include "pmFPA.h"
+#include "pmFPALevel.h"
+#include "pmFPAview.h"
+#include "pmFPAfile.h"
+
+#include "pmPeaks.h"
+#include "pmMoments.h"
+#include "pmGrowthCurve.h"
+#include "pmResiduals.h"
+#include "pmTrend2D.h"
+#include "pmPSF.h"
+#include "pmModel.h"
+#include "pmSource.h"
+#include "pmModelClass.h"
+#include "pmSourceIO.h"
+
+// panstarrs-style FITS table output (header + table in 1st extension)
+// this format consists of a header derived from the image header
+// followed by a zero-size matrix, followed by the table data
+
+// this output format is valid for psphot analysis of an image, and does not include calibrated
+// values derived in the DVO database.
+// XXX how do I generate the source tables which I need to send to PSPS?
+
+bool pmSourcesWrite_PS1_CAL_0 (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader,
+                               psMetadata *tableHeader, char *extname, char *xsrcname)
+{
+    PS_ASSERT_PTR_NON_NULL(fits, false);
+    PS_ASSERT_PTR_NON_NULL(sources, false);
+    PS_ASSERT_PTR_NON_NULL(extname, false);
+
+    psArray *table;
+    psMetadata *row;
+    int i;
+    psF32 *PAR, *dPAR;
+    psEllipseAxes axes;
+    psF32 xPos, yPos;
+    psF32 xErr, yErr;
+    psF32 errMag, chisq;
+
+    pmChip *chip = readout->parent->parent;
+    pmFPA  *fpa  = chip->parent;
+    if (!chip->toFPA || !fpa->toTPA || !fpa->toSky) {
+	psWarning ("astrometry calibration is missing, no calibrated coords");
+    }
+
+    bool status1 = false;
+    bool status2 = false;
+    float magOffset = NAN;
+    float exptime = psMetadataLookupF32 (&status1, fpa->concepts, "FPA.EXPOSURE");
+    float zeropt  = psMetadataLookupF32 (&status2, imageHeader, "ZPT_OBS");
+    if (!status1 || !status2 || (exptime == 0.0)) {
+	psWarning ("exposure time or measured zero point not found for a readout, no calibrated mags");
+    } else {
+	magOffset = zeropt + 2.5*log10(exptime);
+    }
+
+    // let's write these out in S/N order
+    sources = psArraySort (sources, pmSourceSortBySN);
+
+    table = psArrayAllocEmpty (sources->n);
+
+    // we write out PSF-fits for all sources, regardless of quality.  the source flags tell us the state
+    for (i = 0; i < sources->n; i++) {
+        pmSource *source = (pmSource *) sources->data[i];
+	if (source->seq == -1) {
+	    source->seq = i;
+	}
+
+        // no difference between PSF and non-PSF model
+        pmModel *model = source->modelPSF;
+
+        if (model != NULL) {
+            PAR = model->params->data.F32;
+            dPAR = model->dparams->data.F32;
+            xPos = PAR[PM_PAR_XPOS];
+            yPos = PAR[PM_PAR_YPOS];
+            xErr = dPAR[PM_PAR_XPOS];
+            yErr = dPAR[PM_PAR_YPOS];
+	    if (isfinite(PAR[PM_PAR_SXX]) && isfinite(PAR[PM_PAR_SXX]) && isfinite(PAR[PM_PAR_SXX])) {
+		axes = pmPSF_ModelToAxes (PAR, 20.0);
+	    } else {
+		axes.major = NAN;
+		axes.minor = NAN;
+		axes.theta = NAN;
+	    }
+	    chisq = model->chisq;
+
+	    // need to determine the PSF photometry error: source->errMag is the error on the 'best' model mag.
+	    errMag = model->dparams->data.F32[PM_PAR_I0] / model->params->data.F32[PM_PAR_I0];
+        } else {
+            xPos = source->peak->xf;
+            yPos = source->peak->yf;
+            xErr = source->peak->dx;
+            yErr = source->peak->dy;
+            axes.major = NAN;
+            axes.minor = NAN;
+            axes.theta = NAN;
+	    chisq = NAN;
+	    errMag = NAN;
+        }
+
+	float calMag = source->psfMag + magOffset;
+        float peakMag = (source->peak->flux > 0) ? -2.5*log10(source->peak->flux) : NAN;
+        psS16 nImageOverlap = 1;
+
+	// generate RA,DEC
+	psPlane ptCH, ptFP, ptTP;
+	psSphere ptSky;
+
+	ptCH.x = xPos;
+	ptCH.y = yPos;
+	if (chip->toFPA && fpa->toTPA && fpa->toSky) {
+	    psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH);
+	    psPlaneTransformApply (&ptTP, fpa->toTPA, &ptFP);
+	    psDeproject (&ptSky, &ptTP, fpa->toSky);
+	} else {
+	    ptSky.r = NAN;
+	    ptSky.d = NAN;
+	}
+
+        row = psMetadataAlloc ();
+        psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET",         PS_DATA_U32, "IPP detection identifier index",             source->seq);
+        psMetadataAdd (row, PS_LIST_TAIL, "RA_PSF",           PS_DATA_F32, "PSF RA coordinate (degrees)",                ptSky.r*PS_DEG_RAD);
+        psMetadataAdd (row, PS_LIST_TAIL, "DEC_PSF",          PS_DATA_F32, "PSF DEC coordinate (degrees)",               ptSky.d*PS_DEG_RAD);
+	// XXX need to do the error propagation correctly..
+        // psMetadataAdd (row, PS_LIST_TAIL, "RA_PSF_SIG",       PS_DATA_F32, "Sigma of PSF fit RA",                     dRA);
+        // psMetadataAdd (row, PS_LIST_TAIL, "DEC_PSF_SIG",      PS_DATA_F32, "Sigma of PSF fit DEC",                    dDEC);
+
+        psMetadataAdd (row, PS_LIST_TAIL, "X_PSF",            PS_DATA_F32, "PSF x coordinate",                           xPos);
+        psMetadataAdd (row, PS_LIST_TAIL, "Y_PSF",            PS_DATA_F32, "PSF y coordinate",                           yPos);
+        psMetadataAdd (row, PS_LIST_TAIL, "X_PSF_SIG",        PS_DATA_F32, "Sigma in PSF x coordinate",                  xErr);
+        psMetadataAdd (row, PS_LIST_TAIL, "Y_PSF_SIG",        PS_DATA_F32, "Sigma in PSF y coordinate",                  yErr);
+        psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG",      PS_DATA_F32, "Calibrated Magnitude from PSF Fit",          calMag);
+        psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG",  PS_DATA_F32, "Calibrated Magnitude Error",                 errMag);
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_MAG",     PS_DATA_F32, "PSF fit instrumental magnitude",             source->psfMag);
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_MAG_SIG", PS_DATA_F32, "Sigma of PSF instrumental magnitude",        errMag);
+        psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude",           peakMag);
+        psMetadataAdd (row, PS_LIST_TAIL, "SKY",              PS_DATA_F32, "Sky level",                                  source->sky);
+        psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA",        PS_DATA_F32, "Sigma of sky level",                         source->skyErr);
+
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_CHISQ",        PS_DATA_F32,  "Chisq of PSF-fit",                          chisq);
+        psMetadataAdd (row, PS_LIST_TAIL, "CR_NSIGMA",        PS_DATA_F32,  "Nsigma deviations from PSF to CF",          source->crNsigma);
+        psMetadataAdd (row, PS_LIST_TAIL, "EXT_NSIGMA",       PS_DATA_F32,  "Nsigma deviations from PSF to EXT",         source->extNsigma);
+
+	// EXT_NSIGMA will be NAN if: 1) contour ellipse is imaginary; 2) source is not
+	// subtracted
+
+	// CR_NSIGMA will be NAN if: 1) source is not subtracted; 2) source is on the image
+	// edge; 3) any pixels in the 3x3 peak region are masked; 
+
+        // XXX these should be major and minor, not 'x' and 'y'
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_WIDTH_X",      PS_DATA_F32, "PSF width in x coordinate",                  axes.major);
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_WIDTH_Y",      PS_DATA_F32, "PSF width in y coordinate",                  axes.minor);
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_THETA",        PS_DATA_F32, "PSF orientation angle",                      axes.theta);
+        psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF",           PS_DATA_F32, "PSF coverage/quality factor",                source->pixWeight);
+
+        // XXX not sure how to get this : need to load Nimages with weight?
+        psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES",         PS_DATA_U16, "Number of frames overlapping source center", nImageOverlap);
+        psMetadataAdd (row, PS_LIST_TAIL, "FLAGS",            PS_DATA_U16, "psphot analysis flags",                      source->mode);
+
+        psArrayAdd (table, 100, row);
+        psFree (row);
+    }
+
+    if (table->n == 0) {
+        psFitsWriteBlank (fits, tableHeader, extname);
+        psFree (table);
+        return true;
+    }
+
+    psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname);
+    if (!psFitsWriteTable (fits, tableHeader, table, extname)) {
+        psError(PS_ERR_IO, false, "writing ext data %s\n", extname);
+        psFree(table);
+        return false;
+    }
+    psFree (table);
+
+    return true;
+}
+
+// read in a readout from the fits file
+psArray *pmSourcesRead_PS1_CAL_0 (psFits *fits, psMetadata *header)
+{
+    PS_ASSERT_PTR_NON_NULL(fits, false);
+    PS_ASSERT_PTR_NON_NULL(header, false);
+
+    bool status;
+    psF32 *PAR, *dPAR;
+    psEllipseAxes axes;
+
+    // define PSF model type
+    int modelType = pmModelClassGetType ("PS_MODEL_GAUSS");
+
+    char *PSF_NAME = psMetadataLookupStr (&status, header, "PSF_NAME");
+    if (PSF_NAME != NULL) {
+        modelType = pmModelClassGetType (PSF_NAME);
+    }
+    assert (modelType > -1);
+
+    // XXX need to look up the XSRCNAME entries
+
+    // validate a single row of the table (must match SMP)
+
+    // XXX test return values
+
+    // XXX we have a memory problem, which is illustrated here: if I allocate the sources array
+    // (line 1) before freeing the table, the data is not really freed (but not a leak?)  if I
+    // allocate the sources array *after* freeing the table, the data is actually freed
+    // psArray *fooSources = psArrayAllocEmpty (10000);
+    // psFree (table);
+    // return (fooSources);
+
+
+    // We get the size of the table, and allocate the array of sources first because the table is large and
+    // ephemeral --- when the table gets blown away, whatever is allocated after the table is read.  In fact,
+    // it's better to read the table row by row.
+    long numSources = psFitsTableSize(fits); // Number of sources in table
+    psArray *sources = psArrayAlloc(numSources); // Array of sources, to return
+
+    // convert the table to the pmSource entries
+    // XXX need to chooose PSF vs EXT, based on type?
+    for (int i = 0; i < numSources; i++) {
+        psMetadata *row = psFitsReadTableRow(fits, i); // Table row
+
+        pmSource *source = pmSourceAlloc ();
+        pmModel *model = pmModelAlloc (modelType);
+        source->modelPSF  = model;
+        source->type = PM_SOURCE_TYPE_STAR;
+
+        // NOTE: A SEGV here because "model" is NULL is probably caused by not initialising the models.
+        PAR = model->params->data.F32;
+        dPAR = model->dparams->data.F32;
+
+        source->seq       = psMetadataLookupU32 (&status, row, "IPP_IDET");
+        PAR[PM_PAR_XPOS]  = psMetadataLookupF32 (&status, row, "X_PSF");
+        PAR[PM_PAR_YPOS]  = psMetadataLookupF32 (&status, row, "Y_PSF");
+        dPAR[PM_PAR_XPOS] = psMetadataLookupF32 (&status, row, "X_PSF_SIG");
+        dPAR[PM_PAR_YPOS] = psMetadataLookupF32 (&status, row, "Y_PSF_SIG");
+        axes.major        = psMetadataLookupF32 (&status, row, "PSF_WIDTH_X");
+        axes.minor        = psMetadataLookupF32 (&status, row, "PSF_WIDTH_Y");
+        axes.theta        = psMetadataLookupF32 (&status, row, "PSF_THETA");
+
+        PAR[PM_PAR_SKY]   = psMetadataLookupF32 (&status, row, "SKY");
+        dPAR[PM_PAR_SKY]  = psMetadataLookupF32 (&status, row, "SKY_SIGMA");
+        source->sky = PAR[PM_PAR_SKY];
+        source->skyErr = dPAR[PM_PAR_SKY];
+
+        // XXX use these to determine PAR[PM_PAR_I0]?
+        source->psfMag    = psMetadataLookupF32 (&status, row, "PSF_INST_MAG");
+        source->errMag    = psMetadataLookupF32 (&status, row, "PSF_INST_MAG_SIG");
+	PAR[PM_PAR_I0]    = (isfinite(source->psfMag)) ? pow(10.0, -0.4*source->psfMag) : NAN;
+	dPAR[PM_PAR_I0]   = (isfinite(source->psfMag)) ? PAR[PM_PAR_I0] * source->errMag : NAN;
+
+        pmPSF_AxesToModel (PAR, axes);
+
+        float lflux = psMetadataLookupF32 (&status, row, "PEAK_FLUX_AS_MAG");
+        float flux = (isfinite(lflux)) ? pow(10.0, -0.4*lflux) : NAN;
+        source->peak = pmPeakAlloc(PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], flux, PM_PEAK_LONE);
+        source->peak->flux = flux;
+
+        source->pixWeight = psMetadataLookupF32 (&status, row, "PSF_QF");
+
+	// note that some older versions used PSF_PROBABILITY: this was not well defined.
+        model->chisq      = psMetadataLookupF32 (&status, row, "PSF_CHISQ");
+        source->crNsigma  = psMetadataLookupF32 (&status, row, "CR_NSIGMA");
+        source->extNsigma = psMetadataLookupF32 (&status, row, "EXT_NSIGMA");
+
+        source->mode      = psMetadataLookupU16 (&status, row, "FLAGS");
+        assert (status);
+
+        // XXX other values saved but not loaded?
+        // psMetadataLookupS64 (&status, row, "IPP_IDET");
+        // psMetadataLookupF32 (&status, row, "N_FRAMES");
+
+        sources->data[i] = source;
+        psFree(row);
+    }
+
+    return sources;
+}
+
+bool pmSourcesWrite_PS1_CAL_0_XSRC (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname, psMetadata *recipe)
+{
+
+    bool status;
+    psArray *table;
+    psMetadata *row;
+    psF32 *PAR, *dPAR;
+    psF32 xPos, yPos;
+    psF32 xErr, yErr;
+
+    pmChip *chip = readout->parent->parent;
+    pmFPA  *fpa  = chip->parent;
+    if (!chip->toFPA || !fpa->toTPA || !fpa->toSky) {
+	psWarning ("astrometry calibration is missing, no calibrated coords");
+    }
+
+    bool calMags = false;
+    bool status1 = false;
+    bool status2 = false;
+    float magOffset = NAN;
+    float exptime = psMetadataLookupF32 (&status1, fpa->concepts, "FPA.EXPOSURE");
+    float zeropt  = psMetadataLookupF32 (&status2, imageHeader, "ZPT_OBS");
+    if (!status1 || !status2 || (exptime == 0.0)) {
+	psWarning ("exposure time or measured zero point not found for a readout, no calibrated mags");
+    } else {
+	magOffset = zeropt + 2.5*log10(exptime);
+	calMags = true;
+    }
+
+    // create a header to hold the output data
+    psMetadata *outhead = psMetadataAlloc ();
+
+    // write the links to the image header
+    psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "xsrc table extension", extname);
+
+    // let's write these out in S/N order
+    sources = psArraySort (sources, pmSourceSortBySN);
+
+    table = psArrayAllocEmpty (sources->n);
+
+    // which extended source analyses should we perform?
+    bool doPetrosian    = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN");
+    bool doIsophotal    = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ISOPHOTAL");
+    bool doAnnuli       = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANNULI");
+    bool doKron         = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_KRON");
+
+    psVector *radialBinsLower = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER");
+    psVector *radialBinsUpper = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER");
+    assert (radialBinsLower->n == radialBinsUpper->n);
+
+    // we write out all sources, regardless of quality.  the source flags tell us the state
+    for (int i = 0; i < sources->n; i++) {
+	// skip source if it is not a ext sourc
+	// XXX we have two places that extended source parameters are measured:
+	// psphotExtendedSources, which measures the aperture-like parameters and (potentially) the psf-convolved extended source models,
+	// psphotFitEXT, which does the simple extended source model fit (not psf-convolved)
+	// should we require both?
+
+	pmSource *source = sources->data[i];
+
+	// skip sources without measurements
+	if (source->extpars == NULL) continue;
+
+	// we require a PSF model fit (ignore the real crud)
+	pmModel *model = source->modelPSF;
+	if (model == NULL) continue;
+
+	// XXX I need to split the extended models from the extended aperture measurements
+	PAR = model->params->data.F32;
+	dPAR = model->dparams->data.F32;
+	xPos = PAR[PM_PAR_XPOS];
+	yPos = PAR[PM_PAR_YPOS];
+	xErr = dPAR[PM_PAR_XPOS];
+	yErr = dPAR[PM_PAR_YPOS];
+
+	// generate RA,DEC
+	psPlane ptCH, ptFP, ptTP;
+	psSphere ptSky;
+
+	ptCH.x = xPos;
+	ptCH.y = yPos;
+	if (chip->toFPA && fpa->toTPA && fpa->toSky) {
+	    psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH);
+	    psPlaneTransformApply (&ptTP, fpa->toTPA, &ptFP);
+	    psDeproject (&ptSky, &ptTP, fpa->toSky);
+	} else {
+	    ptSky.r = NAN;
+	    ptSky.d = NAN;
+	}
+
+        row = psMetadataAlloc ();
+
+        // XXX we are not writing out the mode (flags) or the type (psf, ext, etc)
+        psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET",         PS_DATA_U32, "IPP detection identifier index",             source->seq);
+        psMetadataAdd (row, PS_LIST_TAIL, "RA_EXT",           PS_DATA_F32, "EXT model RA (degrees)",                     ptSky.r*PS_DEG_RAD);
+        psMetadataAdd (row, PS_LIST_TAIL, "DEC_EXT",          PS_DATA_F32, "EXT model DEC (degrees)",                    ptSky.d*PS_DEG_RAD);
+
+        psMetadataAdd (row, PS_LIST_TAIL, "X_EXT",            PS_DATA_F32, "EXT model x coordinate",                     xPos);
+        psMetadataAdd (row, PS_LIST_TAIL, "Y_EXT",            PS_DATA_F32, "EXT model y coordinate",                     yPos);
+        psMetadataAdd (row, PS_LIST_TAIL, "X_EXT_SIG",        PS_DATA_F32, "Sigma in EXT x coordinate",                  xErr);
+        psMetadataAdd (row, PS_LIST_TAIL, "Y_EXT_SIG",        PS_DATA_F32, "Sigma in EXT y coordinate",                  yErr);
+
+	// Petrosian measurements
+	// XXX insert header data: petrosian ref radius, flux ratio
+	if (doPetrosian) {
+	    pmSourcePetrosianValues *petrosian = source->extpars->petrosian;
+	    if (petrosian) {
+		if (calMags) {
+		    psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_CAL",    PS_DATA_F32, "Petrosian Magnitude (calibrated)", petrosian->mag + magOffset);
+		} else {
+		    psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_INST",   PS_DATA_F32, "Petrosian Magnitude (instrumental)", petrosian->mag);
+		}
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_ERR",    PS_DATA_F32, "Petrosian Magnitude Error", petrosian->magErr);
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS",     PS_DATA_F32, "Petrosian Radius",          petrosian->rad);
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_ERR", PS_DATA_F32, "Petrosian Radius Error",    petrosian->radErr);
+	    } else {
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG",        PS_DATA_F32, "Petrosian Magnitude",       NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_ERR",    PS_DATA_F32, "Petrosian Magnitude Error", NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS",     PS_DATA_F32, "Petrosian Radius",          NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_ERR", PS_DATA_F32, "Petrosian Radius Error",    NAN);
+	    }
+	} 
+
+	// Kron measurements
+	if (doKron) {
+	    pmSourceKronValues *kron = source->extpars->kron;
+	    if (kron) {
+		if (calMags) {
+		    psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_CAL",  PS_DATA_F32, "Kron Magnitude",     kron->mag + magOffset);
+		} else {
+		    psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_INST", PS_DATA_F32, "Kron Magnitude",     kron->mag);
+		}
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_ERR",    PS_DATA_F32, "Kron Magnitude Error", kron->magErr);
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS",     PS_DATA_F32, "Kron Radius",          kron->rad);
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS_ERR", PS_DATA_F32, "Kron Radius Error",    kron->radErr);
+	    } else {
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG",        PS_DATA_F32, "Kron Magnitude",       NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_ERR",    PS_DATA_F32, "Kron Magnitude Error", NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS",     PS_DATA_F32, "Kron Radius",          NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS_ERR", PS_DATA_F32, "Kron Radius Error",    NAN);
+	    }
+	}
+
+	// Isophot measurements
+	// XXX insert header data: isophotal level
+	if (doIsophotal) {
+	    pmSourceIsophotalValues *isophot = source->extpars->isophot;
+	    if (isophot) {
+		if (calMags) {
+		    psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_CAL",    PS_DATA_F32, "Isophot Magnitude (calibrated)",   isophot->mag + magOffset);
+		} else {
+		    psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_INST",   PS_DATA_F32, "Isophot Magnitude (uncalibrated)", isophot->mag);
+		}
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_ERR",    PS_DATA_F32, "Isophot Magnitude Error", isophot->magErr);
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS",     PS_DATA_F32, "Isophot Radius",          isophot->rad);
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS_ERR", PS_DATA_F32, "Isophot Radius Error",    isophot->radErr);
+	    } else {
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG",        PS_DATA_F32, "Isophot Magnitude",       NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_ERR",    PS_DATA_F32, "Isophot Magnitude Error", NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS",     PS_DATA_F32, "Isophot Radius",          NAN);
+		psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS_ERR", PS_DATA_F32, "Isophot Radius Error",    NAN);
+	    }
+	}
+
+	// Flux Annuli
+	if (doAnnuli) {
+	    pmSourceAnnuli *annuli = source->extpars->annuli;
+	    if (annuli) {
+		psVector *fluxVal = annuli->flux;
+		psVector *fluxErr = annuli->fluxErr;
+		psVector *fluxVar = annuli->fluxVar;
+
+		for (int j = 0; j < fluxVal->n; j++) {
+		    char name[32];
+		    sprintf (name, "FLUX_VAL_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux value in annulus", fluxVal->data.F32[j]);
+		    sprintf (name, "FLUX_ERR_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux error in annulus", fluxErr->data.F32[j]);
+		    sprintf (name, "FLUX_VAR_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux stdev in annulus", fluxVar->data.F32[j]);
+		} 
+	    } else {
+		for (int j = 0; j < radialBinsLower->n; j++) {
+		    char name[32];
+		    sprintf (name, "FLUX_VAL_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux value in annulus", NAN);
+		    sprintf (name, "FLUX_ERR_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux error in annulus", NAN);
+		    sprintf (name, "FLUX_VAR_R_%02d", j);
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux stdev in annulus", NAN);
+		} 
+	    }
+	}
+
+	psArrayAdd (table, 100, row);
+	psFree (row);
+    }
+
+    if (table->n == 0) {
+	psFitsWriteBlank (fits, outhead, extname);
+	psFree (outhead);
+	psFree (table);
+	return true;
+    }
+
+    psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname);
+    if (!psFitsWriteTable (fits, outhead, table, extname)) {
+	psError(PS_ERR_IO, false, "writing ext data %s\n", extname);
+	psFree (outhead);
+	psFree(table);
+	return false;
+    }
+    psFree (outhead);
+    psFree (table);
+
+    return true;
+}
+
+bool pmSourcesWrite_PS1_CAL_0_XFIT (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, char *extname)
+{
+
+    psArray *table;
+    psMetadata *row;
+    psF32 *PAR, *dPAR;
+    psEllipseAxes axes;
+    psF32 xPos, yPos;
+    psF32 xErr, yErr;
+    char name[64];
+
+    pmChip *chip = readout->parent->parent;
+    pmFPA  *fpa  = chip->parent;
+    if (!chip->toFPA || !fpa->toTPA || !fpa->toSky) {
+	psWarning ("astrometry calibration is missing, no calibrated coords");
+    }
+
+    bool status1 = false;
+    bool status2 = false;
+    float magOffset = NAN;
+    float exptime = psMetadataLookupF32 (&status1, fpa->concepts, "FPA.EXPOSURE");
+    float zeropt  = psMetadataLookupF32 (&status2, imageHeader, "ZPT_OBS");
+    if (!status1 || !status2 || (exptime == 0.0)) {
+	psWarning ("exposure time or measured zero point not found for a readout, no calibrated mags");
+    } else {
+	magOffset = zeropt + 2.5*log10(exptime);
+    }
+
+    // create a header to hold the output data
+    psMetadata *outhead = psMetadataAlloc ();
+
+    // write the links to the image header
+    psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "xsrc table extension", extname);
+
+    // let's write these out in S/N order
+    sources = psArraySort (sources, pmSourceSortBySN);
+
+    // we are writing one row per model; we need to write out same number of columns for each row: find the max Nparams
+    int nParamMax = 0;
+    for (int i = 0; i < sources->n; i++) {
+	pmSource *source = sources->data[i];
+	if (source->modelFits == NULL) continue;
+	for (int j = 0; j < source->modelFits->n; j++) {
+	    pmModel *model = source->modelFits->data[j];
+	    assert (model);
+	    nParamMax = PS_MAX (nParamMax, model->params->n);
+	}
+    }
+
+    table = psArrayAllocEmpty (sources->n);
+
+    // we write out all sources, regardless of quality.  the source flags tell us the state
+    for (int i = 0; i < sources->n; i++) {
+
+	pmSource *source = sources->data[i];
+
+	// XXX if no model fits are saved, write out modelEXT?
+	if (source->modelFits == NULL) continue;
+
+	// We have multiple sources : need to flag the one used to subtract the light (the 'best' model)
+	for (int j = 0; j < source->modelFits->n; j++) {
+
+	    // choose the convolved EXT model, if available, otherwise the simple one
+	    pmModel *model = source->modelFits->data[j];
+	    assert (model);
+
+	    PAR = model->params->data.F32;
+	    dPAR = model->dparams->data.F32;
+	    xPos = PAR[PM_PAR_XPOS];
+	    yPos = PAR[PM_PAR_YPOS];
+	    xErr = dPAR[PM_PAR_XPOS];
+	    yErr = dPAR[PM_PAR_YPOS];
+
+	    axes = pmPSF_ModelToAxes (PAR, 20.0);
+
+	    // generate RA,DEC
+	    psPlane ptCH, ptFP, ptTP;
+	    psSphere ptSky;
+
+	    ptCH.x = xPos;
+	    ptCH.y = yPos;
+	    if (chip->toFPA && fpa->toTPA && fpa->toSky) {
+		psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH);
+		psPlaneTransformApply (&ptTP, fpa->toTPA, &ptFP);
+		psDeproject (&ptSky, &ptTP, fpa->toSky);
+	    } else {
+		ptSky.r = NAN;
+		ptSky.d = NAN;
+	    }
+
+	    row = psMetadataAlloc ();
+
+	    // XXX we are not writing out the mode (flags) or the type (psf, ext, etc)
+	    psMetadataAddU32 (row, PS_LIST_TAIL, "IPP_IDET",         0, "IPP detection identifier index",             source->seq);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "RA_EXT",           0, "EXT model RA (degrees)",                     ptSky.r);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "DEC_EXT",          0, "EXT model DEC (degrees)",                    ptSky.d);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "X_EXT",            0, "EXT model x coordinate",                     xPos);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "Y_EXT",            0, "EXT model y coordinate",                     yPos);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "X_EXT_SIG",        0, "Sigma in EXT x coordinate",                  xErr);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "Y_EXT_SIG",        0, "Sigma in EXT y coordinate",                  yErr);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_CAL_MAG",      0, "EXT fit calibrated magnitude",               model->mag + magOffset);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_INST_MAG",     0, "EXT fit instrumental magnitude",             model->mag);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_INST_MAG_SIG", 0, "Sigma of PSF instrumental magnitude",        model->magErr);
+
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS",          0, "number of model parameters",                 model->params->n);
+	    psMetadataAddStr (row, PS_LIST_TAIL, "MODEL_TYPE",       0, "name of model",                              pmModelClassGetName (model->type));
+
+	    // XXX these should be major and minor, not 'x' and 'y'
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_WIDTH_MAJ",    0, "EXT width in x coordinate",                  axes.major);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_WIDTH_MIN",    0, "EXT width in y coordinate",                  axes.minor);
+	    psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_THETA",        0, "EXT orientation angle",                      axes.theta);
+
+	    // write out the other generic parameters
+	    for (int k = 0; k < nParamMax; k++) {
+		if (k == PM_PAR_I0) continue;
+		if (k == PM_PAR_SKY) continue;
+		if (k == PM_PAR_XPOS) continue;
+		if (k == PM_PAR_YPOS) continue;
+		if (k == PM_PAR_SXX) continue;
+		if (k == PM_PAR_SXY) continue;
+		if (k == PM_PAR_SYY) continue;
+
+		snprintf (name, 64, "EXT_PAR_%02d", k);
+
+		if (k < model->params->n) {
+		    psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "", model->params->data.F32[k]);
+		} else {
+		    psMetadataAddF32 (row, PS_LIST_TAIL, name, PS_DATA_F32, "", NAN);
+		}
+	    }
+
+	    // XXX other parameters which may be set.
+	    // XXX flag / value to define the model
+	    // XXX write out the model type, fit status flags
+
+	    psArrayAdd (table, 100, row);
+	    psFree (row);
+	}
+    }
+
+    if (table->n == 0) {
+	psFitsWriteBlank (fits, outhead, extname);
+	psFree (outhead);
+	psFree (table);
+	return true;
+    }
+
+    psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname);
+    if (!psFitsWriteTable (fits, outhead, table, extname)) {
+	psError(PS_ERR_IO, false, "writing ext data %s\n", extname);
+	psFree (outhead);
+	psFree(table);
+	return false;
+    }
+    psFree (outhead);
+    psFree (table);
+    return true;
+}
Index: branches/bills_081204/psModules/src/objects/pmSourcePlotApResid.c
===================================================================
--- branches/cnb_branch_20081104/psModules/src/objects/pmSourcePlotApResid.c	(revision 20530)
+++ branches/bills_081204/psModules/src/objects/pmSourcePlotApResid.c	(revision 20890)
@@ -4,6 +4,6 @@
  *  @author EAM, IfA
  *
- *  @version $Revision: 1.5 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2007-11-10 01:09:20 $
+ *  @version $Revision: 1.6 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-12-04 00:07:25 $
  *  Copyright 2006 IfA, University of Hawaii
  */
@@ -63,5 +63,5 @@
         return false;
 
-    int kapa = pmKapaOpen (true);
+    int kapa = pmKapaOpen (false);
     if (kapa == -1) {
         psError(PS_ERR_UNKNOWN, true, "failure to open kapa");
Index: branches/bills_081204/psModules/src/objects/pmSourceVisual.c
===================================================================
--- branches/bills_081204/psModules/src/objects/pmSourceVisual.c	(revision 20890)
+++ branches/bills_081204/psModules/src/objects/pmSourceVisual.c	(revision 20890)
@@ -0,0 +1,256 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <pslib.h>
+#include "pmTrend2D.h"
+#include "pmSourceVisual.h"
+
+# if (HAVE_KAPA)
+# include <kapa.h>
+
+// functions used to visualize the analysis as it goes
+// these are invoked by the -visual options
+
+static bool isVisual = false;
+static int kapa1 = -1;
+// static int kapa2 = -1;
+// static int kapa3 = -1;
+
+bool pmSourcePlotPoints3D (int myKapa, Graphdata *graphdata, psVector *xn, psVector *yn, psVector *zn, float theta, float phi);
+
+bool pmSourceSetVisual (bool mode) {
+
+    isVisual = mode;
+    return true;
+}
+
+bool pmSourceVisualPSFModelResid (pmTrend2D *trend, psVector *x, psVector *y, psVector *param, psVector *mask) {
+
+    KapaSection section;  // put the positive profile in one and the residuals in another? 
+
+    Graphdata graphdata;
+
+    if (!isVisual) return true;
+
+    if (kapa1 == -1) {
+        kapa1 = KapaOpenNamedSocket ("kapa", "pmSource:plots");
+	if (kapa1 == -1) {
+	    fprintf (stderr, "failure to open kapa; visual mode disabled\n");
+	    isVisual = false;
+	    return false;
+	}
+    }  
+
+    KapaClearPlots (kapa1);
+    KapaInitGraph (&graphdata);
+
+    float min = +1e32;
+    float max = -1e32;
+    float Min = +1e32;
+    float Max = -1e32;
+
+    psVector *resid = psVectorAlloc (x->n, PS_TYPE_F32);
+    psVector *model = psVectorAlloc (x->n, PS_TYPE_F32);
+
+    for (int i = 0; i < x->n; i++) {
+	model->data.F32[i] = pmTrend2DEval (trend, x->data.F32[i], y->data.F32[i]);
+	resid->data.F32[i] = param->data.F32[i] - model->data.F32[i];
+	if (mask->data.U8[i]) continue;
+	min = PS_MIN (min, resid->data.F32[i]);
+	max = PS_MAX (max, resid->data.F32[i]);
+	Min = PS_MIN (min, param->data.F32[i]);
+	Max = PS_MAX (max, param->data.F32[i]);
+    }
+
+    psVector *xn = psVectorAlloc (x->n, PS_TYPE_F32);
+    psVector *yn = psVectorAlloc (x->n, PS_TYPE_F32);
+    psVector *zn = psVectorAlloc (x->n, PS_TYPE_F32);
+    psVector *Zn = psVectorAlloc (x->n, PS_TYPE_F32);
+    psVector *Fn = psVectorAlloc (x->n, PS_TYPE_F32);
+    for (int i = 0; i < x->n; i++) {
+	xn->data.F32[i] = x->data.F32[i] / 5000.0;
+	yn->data.F32[i] = y->data.F32[i] / 5000.0;
+	zn->data.F32[i] = (resid->data.F32[i] - min) / (max - min);
+	Zn->data.F32[i] = (param->data.F32[i] - Min) / (Max - Min);
+	Fn->data.F32[i] = (model->data.F32[i] - Min) / (Max - Min);
+    }
+
+    // view 1 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.0;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a1");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, zn, 30.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    // view 2 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.5;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a2");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, zn, -60.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    // view 3 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.0;
+    section.y = 0.33;
+    section.name = NULL;
+    psStringAppend (&section.name, "a3");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, Zn, 30.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    // view 4 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.5;
+    section.y = 0.33;
+    section.name = NULL;
+    psStringAppend (&section.name, "a4");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, Zn, -60.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    // view 5 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.0;
+    section.y = 0.66;
+    section.name = NULL;
+    psStringAppend (&section.name, "a5");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, Fn, 30.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    // view 6 on resid
+    section.dx = 0.5;
+    section.dy = 0.33;
+    section.x = 0.5;
+    section.y = 0.66;
+    section.name = NULL;
+    psStringAppend (&section.name, "a6");
+    KapaSetSection (kapa1, &section);
+    psFree (section.name);
+    pmSourcePlotPoints3D (kapa1, &graphdata, xn, yn, Fn, -60.0*PS_RAD_DEG, -15.0*PS_RAD_DEG);
+
+    psFree (resid);
+
+    psFree (xn);
+    psFree (yn);
+    psFree (zn);
+    psFree (Zn);
+
+    // pause and wait for user input:
+    // continue, save (provide name), ??
+    char key[10];
+    fprintf (stdout, "[c]ontinue? ");
+    if (!fgets(key, 8, stdin)) {
+	psWarning("Unable to read option");
+    }
+    return true;
+}
+
+// send in normalized points
+bool pmSourcePlotPoints3D (int myKapa, Graphdata *graphdata, psVector *xn, psVector *yn, psVector *zn, float theta, float phi) {
+
+    psVector *xv = psVectorAlloc (PS_MAX(6, 2*xn->n), PS_TYPE_F32);
+    psVector *yv = psVectorAlloc (PS_MAX(6, 2*xn->n), PS_TYPE_F32);
+    psVector *zv = psVectorAlloc (PS_MAX(6, 2*xn->n), PS_TYPE_F32);
+
+    graphdata->xmin = +1e32;
+    graphdata->xmax = -1e32;
+    graphdata->ymin = +1e32;
+    graphdata->ymax = -1e32;
+
+    for (int i = 0; i < xn->n; i++) {
+	xv->data.F32[2*i+0] = +xn->data.F32[i]*cos(theta) + yn->data.F32[i]*sin(theta)*cos(phi) + zn->data.F32[i]*sin(theta)*sin(phi);
+	yv->data.F32[2*i+0] = -xn->data.F32[i]*sin(theta) + yn->data.F32[i]*cos(theta)*cos(phi) + zn->data.F32[i]*cos(theta)*sin(phi);
+	zv->data.F32[2*i+0] = -yn->data.F32[i]*sin(phi)   + zn->data.F32[i]*cos(phi);
+	xv->data.F32[2*i+1] = +xn->data.F32[i]*cos(theta) + yn->data.F32[i]*sin(theta)*cos(phi);
+	yv->data.F32[2*i+1] = -xn->data.F32[i]*sin(theta) + yn->data.F32[i]*cos(theta)*cos(phi);
+	zv->data.F32[2*i+1] = -yn->data.F32[i]*sin(phi);
+	graphdata->xmin = PS_MIN(graphdata->xmin, xv->data.F32[2*i+0]);
+	graphdata->xmax = PS_MAX(graphdata->xmax, xv->data.F32[2*i+0]);
+	graphdata->ymin = PS_MIN(graphdata->ymin, zv->data.F32[2*i+0]);
+	graphdata->ymax = PS_MAX(graphdata->ymax, zv->data.F32[2*i+0]);
+	graphdata->xmin = PS_MIN(graphdata->xmin, xv->data.F32[2*i+1]);
+	graphdata->xmax = PS_MAX(graphdata->xmax, xv->data.F32[2*i+1]);
+	graphdata->ymin = PS_MIN(graphdata->ymin, zv->data.F32[2*i+1]);
+	graphdata->ymax = PS_MAX(graphdata->ymax, zv->data.F32[2*i+1]);
+    }
+    xv->n = xn->n;
+
+    // examine sources to set data range
+    KapaSetLimits (myKapa, graphdata);
+
+    // KapaSetFont (myKapa, "helvetica", 14);
+    // KapaBox (myKapa, graphdata);
+    // KapaSendLabel (myKapa, "&ss&h_x| (pixels)", KAPA_LABEL_XM);
+    // KapaSendLabel (myKapa, "&ss&h_y| (pixels)", KAPA_LABEL_YM);
+
+    graphdata->color = KapaColorByName ("black");
+    graphdata->ptype = 100;
+    graphdata->size = 0.5;
+    graphdata->style = 2;
+    KapaPrepPlot (myKapa, xv->n, graphdata);
+    KapaPlotVector (myKapa, xv->n, xv->data.F32, "x");
+    KapaPlotVector (myKapa, xv->n, zv->data.F32, "y");
+
+    graphdata->color = KapaColorByName ("blue");
+    graphdata->ptype = 0;
+    graphdata->size = 1.5;
+    graphdata->style = 2;
+    KapaPrepPlot (myKapa, xv->n, graphdata);
+    KapaPlotVector (myKapa, xv->n, xv->data.F32, "x");
+    KapaPlotVector (myKapa, xv->n, zv->data.F32, "y");
+
+    xv->n = 6;
+
+    // set the three axis lines
+    xv->data.F32[0] = +0.0*cos(theta) + 0.0*sin(theta)*cos(phi) + 0.0*sin(theta)*sin(phi);
+    yv->data.F32[0] = -0.0*sin(theta) + 0.0*cos(theta)*cos(phi) + 0.0*cos(theta)*sin(phi);
+    zv->data.F32[0] =                 - 0.0*sin(phi)            + 0.0*cos(phi);
+    xv->data.F32[1] = +1.0*cos(theta) + 0.0*sin(theta)*cos(phi);
+    yv->data.F32[1] = -1.0*sin(theta) + 0.0*cos(theta)*cos(phi);
+    zv->data.F32[1] =                 - 0.0*sin(phi)            + 0.0*cos(phi);
+
+    xv->data.F32[2] = +0.0*cos(theta) + 0.0*sin(theta)*cos(phi) + 0.0*sin(theta)*sin(phi);
+    yv->data.F32[2] = -0.0*sin(theta) + 0.0*cos(theta)*cos(phi) + 0.0*cos(theta)*sin(phi);
+    zv->data.F32[2] =                 - 0.0*sin(phi)            + 0.0*cos(phi);
+    xv->data.F32[3] = +0.0*cos(theta) + 1.0*sin(theta)*cos(phi);
+    yv->data.F32[3] = -0.0*sin(theta) + 1.0*cos(theta)*cos(phi);
+    zv->data.F32[3] =                 - 1.0*sin(phi)            + 0.0*cos(phi);
+
+    xv->data.F32[4] = +0.0*cos(theta) + 0.0*sin(theta)*cos(phi) + 1.0*sin(theta)*sin(phi);
+    yv->data.F32[4] = -0.0*sin(theta) + 0.0*cos(theta)*cos(phi) + 1.0*cos(theta)*sin(phi);
+    zv->data.F32[4] =                 - 0.0*sin(phi)            + 1.0*cos(phi);
+    xv->data.F32[5] = +0.0*cos(theta) + 0.0*sin(theta)*cos(phi);
+    yv->data.F32[5] = -0.0*sin(theta) + 0.0*cos(theta)*cos(phi);
+    zv->data.F32[5] =                 - 0.0*sin(phi)            + 0.0*cos(phi);
+
+    graphdata->color = KapaColorByName ("red");
+    graphdata->ptype = 100;
+    graphdata->size = 0.5;
+    graphdata->style = 2;
+    KapaPrepPlot (myKapa, xv->n, graphdata);
+    KapaPlotVector (myKapa, xv->n, xv->data.F32, "x");
+    KapaPlotVector (myKapa, xv->n, zv->data.F32, "y");
+
+    psFree (xv);
+    psFree (yv);
+    psFree (zv);
+
+    return true;
+}
+
+# else
+# endif
Index: branches/bills_081204/psModules/src/objects/pmSourceVisual.h
===================================================================
--- branches/bills_081204/psModules/src/objects/pmSourceVisual.h	(revision 20890)
+++ branches/bills_081204/psModules/src/objects/pmSourceVisual.h	(revision 20890)
@@ -0,0 +1,21 @@
+/* @file  pmKapaPlots.h
+ * @brief functions to make plots with the external program 'kapa' 
+ *
+ * @author EAM, IfA
+ *
+ * @version $Revision: 1.1 $ $Name: not supported by cvs2svn $
+ * @date $Date: 2008-11-08 01:52:34 $
+ * Copyright 2006 Institute for Astronomy, University of Hawaii
+ */
+
+#ifndef PM_SOURCE_VISUAL_H
+#define PM_SOURCE_VISUAL_H
+
+/// @addtogroup Extras Miscellaneous Funtions
+/// @{
+
+bool pmSourceSetVisual (bool mode);
+bool pmSourceVisualPSFModelResid (pmTrend2D *trend, psVector *x, psVector *y, psVector *param, psVector *mask);
+
+/// @}
+#endif // PM_KAPA_PLOTS_H
Index: branches/bills_081204/psModules/src/psmodules.h
===================================================================
--- branches/cnb_branch_20081104/psModules/src/psmodules.h	(revision 20530)
+++ branches/bills_081204/psModules/src/psmodules.h	(revision 20890)
@@ -72,4 +72,5 @@
 // #include <pmSkySubtract.h>
 #include <pmDark.h>
+#include <pmRemnance.h>
 
 // the following headers are from psModule:astrom
@@ -81,4 +82,5 @@
 #include <pmAstrometryRefstars.h>
 #include <pmAstrometryDistortion.h>
+#include <pmAstrometryVisual.h>
 
 // the following headers are from psModule:imcombine
@@ -121,4 +123,5 @@
 #include <pmModelUtils.h>
 #include <pmSourcePhotometry.h>
+#include <pmSourceVisual.h>
 
 // The following headers are from random locations, here because they cross bounds
Index: branches/bills_081204/psastro/src/Makefile.am
===================================================================
--- branches/cnb_branch_20081104/psastro/src/Makefile.am	(revision 20530)
+++ branches/bills_081204/psastro/src/Makefile.am	(revision 20890)
@@ -53,6 +53,8 @@
 	psastroErrorCodes.c         \
 	psastroVersion.c            \
+	psastroVisual.c             \
 	psastroDefineFiles.c        \
 	psastroAnalysis.c           \
+	psastroMaskUpdates.c        \
 	psastroAstromGuess.c        \
 	psastroLoadRefstars.c       \
Index: branches/bills_081204/psastro/src/psastro.h
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastro.h	(revision 20530)
+++ branches/bills_081204/psastro/src/psastro.h	(revision 20890)
@@ -36,4 +36,5 @@
 bool              psastroAstromGuess (int *nStars, pmConfig *config);
 bool              psastroAstromGuessCheck (pmConfig *config);
+bool              psastroMaskUpdates (pmConfig *config);
 
 psPlaneDistort   *psPlaneDistortIdentity ();
@@ -50,4 +51,5 @@
 bool              psastroLuminosityFunctionPlot(psVector *lnMag, psVector *Mag, pmLumFunc *lumFunc, pmLumFunc *rawFunc);
 psArray          *psastroRemoveClumps (psArray *input, int scale);
+
 
 // utility functions:
@@ -74,4 +76,18 @@
 psString          psastroVersion(void);
 psString          psastroVersionLong(void);
+
+// psastroVisual functions
+bool psastroSetVisual (bool mode);
+bool psastroVisualClose();
+bool psastroVisualPlotLuminosityFunction (psVector *lnMag, psVector *Mag, pmLumFunc *lumFunc, pmLumFunc *rawFunc);
+bool psastroVisualPlotRawStars (psArray *rawstars, pmFPA *fpa, pmChip *chip, psMetadata *recipe);
+bool psastroVisualPlotRefStars (psArray *refstars, psMetadata *recipe);
+bool psastroVisualPlotRemoveClumps (psArray *input, psImage *count, int scale, float limit);
+bool psastroVisualPlotFixChips (pmFPAfile *input, psVector *xOld, psVector *yOld);
+bool psastroVisualPlotOneChipFit (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe);
+bool psastroVisualPlotAstromGuessCheck (psVector *cornerPo, psVector *cornerQo, psVector *cornerPn, psVector *cornerQn, psVector *cornerPd, psVector *cornerQd);
+bool psastroVisualPlotMosaicOneChip (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe);
+bool psastroVisualPlotCommonScale (pmFPA *fpa, psVector *oldScale);
+bool psastroVisualPlotMosaicMatches (psArray *rawstars, psArray *refstars, psArray *match, int iteration, psMetadata *recipe);
 
 // demo plots
Index: branches/bills_081204/psastro/src/psastroAnalysis.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroAnalysis.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroAnalysis.c	(revision 20890)
@@ -83,4 +83,6 @@
     psastroAstromGuessCheck (config);
 
+    psastroMaskUpdates (config);
+
     // XXX how do we specify stack astrometry?
     // psastroStackAstrom (config, refs);
Index: branches/bills_081204/psastro/src/psastroArguments.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroArguments.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroArguments.c	(revision 20890)
@@ -37,5 +37,5 @@
         psArgumentRemove (N, &argc, argv);
     }
-    
+
     // apply the chip correction based on the reference astrometry?
     if ((N = psArgumentGet (argc, argv, "-fixchips"))) {
@@ -51,7 +51,11 @@
     status = pmConfigFileSetsMD (config->arguments, &argc, argv, "ASTROM.MODEL", "-astrommodel", "-astrommodellist");
     if (status) {
-	// if supplied, assume -fixchips is desired
+        // if supplied, assume -fixchips is desired
         psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.FIX.CHIPS", PS_META_REPLACE, "", true);
     }
+
+    // define the reference astrometry file
+    status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT.MASK", "-mask", "-masklist");
+    status = pmConfigFileSetsMD (config->arguments, &argc, argv, "REFMASK", "-refmask", "-refmasklist");
 
     if ((N = psArgumentGet(argc, argv, "-stats"))) {
@@ -79,4 +83,11 @@
         psArgumentRemove (N, &argc, argv);
         psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.CHIP.MODE", PS_META_REPLACE, "", true);
+    }
+
+    // run in visual mode?
+    if ((N = psArgumentGet (argc, argv, "-visual"))) {
+        psArgumentRemove (N, &argc, argv);
+        psastroSetVisual (true);
+        pmAstromSetVisual (true);
     }
 
Index: branches/bills_081204/psastro/src/psastroAstromGuess.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroAstromGuess.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroAstromGuess.c	(revision 20890)
@@ -142,4 +142,6 @@
 		    psastroDumpRawstars (rawstars, fpa, chip);
 		}
+
+                psastroVisualPlotRawStars(rawstars, fpa, chip, recipe);
 
 		if (psTraceGetLevel("psastro.plot") > 0) {
@@ -265,4 +267,6 @@
     psVector *cornerDn = psVectorAllocEmpty (100, PS_TYPE_F32);
 
+    psVector *cornerMK = psVectorAllocEmpty (100, PS_TYPE_U8);
+
     if (DEBUG) psastroDumpCorners ("corners.up.guess3.dat", "corners.dn.guess3.dat", fpa);
 
@@ -272,4 +276,26 @@
     while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) {
         if (!chip->process || !chip->file_exists || !chip->data_exists) { continue; }
+
+	// XXX we are currently inconsistent with marking the good vs the bad data
+	// psastroChipAstrom sets data_exists to false if the fit is bad.  this is
+	// probably wrong since it implies there is no data!
+
+	// skip chips for which the astrometry failed (NASTRO == 0)
+	if (!chip->cells->n) goto skip_chip;
+	pmCell *cell = chip->cells->data[0];
+	if (!cell) goto skip_chip;
+
+	if (!cell->readouts->n) goto skip_chip;
+	pmReadout *readout = cell->readouts->data[0];
+	if (!readout) goto skip_chip;
+
+	psMetadata *updates = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER");
+	if (!updates) goto skip_chip;
+	
+	int nAstro = psMetadataLookupS32 (&status, updates, "NASTRO");
+	if (!nAstro) goto skip_chip;
+
+	float astError = psMetadataLookupF32 (&status, updates, "CERROR");
+	if (fabs(astError) < 1e-6) goto skip_chip;
 
 	psPlane ptCH, ptFP, ptTP;
@@ -290,4 +316,16 @@
 	psVectorAppend (cornerRn, ptSky.r);
 	psVectorAppend (cornerDn, ptSky.d);
+	psVectorAppend (cornerMK, 0);
+	continue;
+
+    skip_chip:
+	// new corner locations based on the calibrated astrometry
+	psVectorAppend (cornerLn, 0.0);
+	psVectorAppend (cornerMn, 0.0);
+	psVectorAppend (cornerPn, 0.0);
+	psVectorAppend (cornerQn, 0.0);
+	psVectorAppend (cornerRn, 0.0);
+	psVectorAppend (cornerDn, 0.0);
+	psVectorAppend (cornerMK, 1);
     }
 
@@ -314,6 +352,7 @@
     map->y->coeffMask[1][1] = PS_POLY_MASK_SET;
     
-    psVectorFitPolynomial2D (map->x, NULL, 0, cornerPn, NULL, cornerPs, cornerQs);
-    psVectorFitPolynomial2D (map->y, NULL, 0, cornerQn, NULL, cornerPs, cornerQs);
+    // fit the valid chips, mask the invalid chips
+    psVectorFitPolynomial2D (map->x, cornerMK, 1, cornerPn, NULL, cornerPs, cornerQs);
+    psVectorFitPolynomial2D (map->y, cornerMK, 1, cornerQn, NULL, cornerPs, cornerQs);
     
     // apply the linear fit...
@@ -325,9 +364,11 @@
     psVector *cornerQd = (psVector *) psBinaryOp (NULL, cornerQn, "-", cornerQf);
 
+    psastroVisualPlotAstromGuessCheck (cornerPo, cornerQo, cornerPn, cornerQn, cornerPd, cornerQd);
+
     psStats *statsP = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
     psStats *statsQ = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
 
-    psVectorStats (statsP, cornerPd, NULL, NULL, 0);
-    psVectorStats (statsQ, cornerQd, NULL, NULL, 0);
+    psVectorStats (statsP, cornerPd, NULL, cornerMK, 1);
+    psVectorStats (statsQ, cornerQd, NULL, cornerMK, 1);
 
     float angle = atan2 (map->y->coeff[1][0], map->x->coeff[1][0]);
@@ -353,5 +394,5 @@
     psMetadataAddF32 (header, PS_LIST_TAIL, "AST_DS", PS_META_REPLACE, "boresite scatter in DEC (TP units)", statsQ->sampleStdev);
 
-    if (0) {
+    if (DEBUG) {
 	FILE *f = fopen ("corners.dat", "w");
 	for (int i = 0; i < cornerRo->n; i++) {
@@ -370,4 +411,6 @@
     psFree (statsP);
     psFree (statsQ);
+
+    psFree (cornerMK);
 
     psFree (cornerLn);
Index: branches/bills_081204/psastro/src/psastroCleanup.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroCleanup.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroCleanup.c	(revision 20890)
@@ -11,4 +11,6 @@
     pmConceptsDone ();
     pmConfigDone ();
+    psastroVisualClose ();
+    pmAstromVisualClose ();
     fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, stdout, false), "psastro");
     // fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "psastro");
Index: branches/bills_081204/psastro/src/psastroDefineFiles.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroDefineFiles.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroDefineFiles.c	(revision 20890)
@@ -33,4 +33,39 @@
             return NULL;
         }
+    }
+
+    // this step is optional
+    bool REFSTAR_MASK = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK");
+    if (REFSTAR_MASK) {
+
+        if (!psastroDefineFile (config, input->fpa, "PSASTRO.REFMASK", "REFMASK", PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) {
+            psError (PS_ERR_IO, false, "Can't find an input mask file");
+            return NULL;
+        }
+
+        if (!psastroDefineFile (config, input->fpa, "PSASTRO.INPUT.MASK", "INPUT.MASK", PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) {
+            psError (PS_ERR_IO, false, "Can't find an input mask file");
+            return NULL;
+        }
+
+	pmFPAfile *inMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT.MASK");
+	if (!inMask) {
+            psError (PS_ERR_IO, false, "Can't find an input mask file");
+            return NULL;
+        }
+
+	// XXX not yet sure if we need to mosaic or not...
+	pmFPAfile *outMask = pmFPAfileDefineOutputFromFile  (config, inMask, "PSASTRO.OUTPUT.MASK");
+	// pmFPAfile *outMask = pmFPAfileDefineChipMosaic(config, inMask->fpa, "PSASTRO.OUTPUT.MASK");
+	if (!outMask) {
+	    psError (PS_ERR_IO, false, "Can't find the astrometry refstars file definition");
+	    return NULL;
+	}
+	if (outMask->type != PM_FPA_FILE_MASK) {
+	    psError(PS_ERR_IO, true, "%s is not of type %s", "PSASTRO.OUTPUT.MASK", pmFPAfileStringFromType (PM_FPA_FILE_MASK));
+	    return NULL;
+	}
+	outMask->save = true;
+	inMask->freeLevel = outMask->dataLevel;
     }
 
Index: branches/bills_081204/psastro/src/psastroFixChips.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroFixChips.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroFixChips.c	(revision 20890)
@@ -1,4 +1,5 @@
 # include "psastroInternal.h"
 # define NONLIN_TOL 0.001 /* tolerance in pixels */
+# define DEBUG 0
 
 // XXX I think the 'badAstrom' tests may need to be adjusted: see eg the nominal rotation of
@@ -7,4 +8,6 @@
 
     bool status;
+    FILE *f;
+    char *chipName;
 
     bool fixChips = psMetadataLookupBool (&status, config->arguments, "PSASTRO.FIX.CHIPS");
@@ -49,7 +52,7 @@
     }
 
-    // we now have a set of chip solutions and a set of prediction measure the overall
-    // offset and rotation of the two systems by comparing the chip corners, projected onto
-    // the focal plane (all 4 to prevent solutions tied to a single corner)
+    // We now have a set of chip solutions and a set of predictions.  Measure the overall offset and
+    // rotation of the two systems by comparing the chip corners, projected onto the focal plane
+    // (all 4 to prevent solutions tied to a single corner)
 
     psVector *xObs = psVectorAllocEmpty (4*input->fpa->chips->n, PS_TYPE_F32);
@@ -60,7 +63,31 @@
     int nPts = 0;
 
+    if (DEBUG) {
+	f = fopen ("corners.raw.dat", "w");
+	chipName = NULL;
+    }
+
     pmChip *obsChip = NULL;
     while ((obsChip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) {
 	if (!obsChip->process || !obsChip->file_exists || !obsChip->data_exists) { continue; }
+
+	// XXX we are currently inconsistent with marking the good vs the bad data
+	// psastroChipAstrom sets data_exists to false if the fit is bad.  this is
+	// probably wrong since it implies there is no data!
+
+	// skip chips for which the astrometry failed (NASTRO == 0)
+	if (!obsChip->cells->n) continue;
+	pmCell *cell = obsChip->cells->data[0];
+	if (!cell) continue;
+
+	if (!cell->readouts->n) continue;
+	pmReadout *readout = cell->readouts->data[0];
+	if (!readout) continue;
+
+	psMetadata *updates = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER");
+	if (!updates) continue;
+	
+	int nAstro = psMetadataLookupS32 (&status, updates, "NASTRO");
+	if (!nAstro) continue;
 
 	// set the chip astrometry using the astrom file
@@ -77,4 +104,9 @@
 	xRef->data.F32[nPts] = ptFP.x;
 	yRef->data.F32[nPts] = ptFP.y;
+
+	if (DEBUG) {
+	    chipName = psMetadataLookupStr(NULL, obsChip->concepts, "CHIP.NAME");
+	    fprintf (f, "%s  %f %f  %f %f\n", chipName, xObs->data.F32[nPts], yObs->data.F32[nPts], xRef->data.F32[nPts], yRef->data.F32[nPts]);
+	}
 	nPts ++;
 
@@ -86,4 +118,9 @@
 	xRef->data.F32[nPts] = ptFP.x;
 	yRef->data.F32[nPts] = ptFP.y;
+
+	if (DEBUG) {
+	    chipName = psMetadataLookupStr(NULL, obsChip->concepts, "CHIP.NAME");
+	    fprintf (f, "%s  %f %f  %f %f\n", chipName, xObs->data.F32[nPts], yObs->data.F32[nPts], xRef->data.F32[nPts], yRef->data.F32[nPts]);
+	}
 	nPts ++;
 
@@ -95,4 +132,9 @@
 	xRef->data.F32[nPts] = ptFP.x;
 	yRef->data.F32[nPts] = ptFP.y;
+
+	if (DEBUG) {
+	    chipName = psMetadataLookupStr(NULL, obsChip->concepts, "CHIP.NAME");
+	    fprintf (f, "%s  %f %f  %f %f\n", chipName, xObs->data.F32[nPts], yObs->data.F32[nPts], xRef->data.F32[nPts], yRef->data.F32[nPts]);
+	}
 	nPts ++;
 
@@ -104,4 +146,9 @@
 	xRef->data.F32[nPts] = ptFP.x;
 	yRef->data.F32[nPts] = ptFP.y;
+
+	if (DEBUG) {
+	    chipName = psMetadataLookupStr(NULL, obsChip->concepts, "CHIP.NAME");
+	    fprintf (f, "%s  %f %f  %f %f\n", chipName, xObs->data.F32[nPts], yObs->data.F32[nPts], xRef->data.F32[nPts], yRef->data.F32[nPts]);
+	}
 	nPts ++;
 
@@ -109,4 +156,5 @@
     }
     xObs->n = yObs->n = xRef->n = yRef->n = nPts;
+    if (DEBUG) fclose (f);
 	
     psPlaneTransform *map = psPlaneTransformAlloc (1, 1);
@@ -123,5 +171,5 @@
 
 	psVectorClipFitPolynomial2D (map->y, stats, mask, 0xff, yObs, NULL, xRef, yRef);
-	psTrace ("psModules.astrom", 3, "x resid: %f +/- %f (%ld of %ld)\n", stats->clippedMean, stats->clippedStdev, stats->clippedNvalues, yObs->n);
+	psTrace ("psModules.astrom", 3, "y resid: %f +/- %f (%ld of %ld)\n", stats->clippedMean, stats->clippedStdev, stats->clippedNvalues, yObs->n);
     }
 
@@ -130,6 +178,16 @@
     // model transformation
 
-    psFree (xObs);
-    psFree (yObs);
+    if (DEBUG) {
+	f = fopen ("corners.fit.dat", "w");
+	for (int i = 0; i < xObs->n; i++) {
+	    psPlane obsCoord, refCoord;
+	    refCoord.x = xRef->data.F32[i];
+	    refCoord.y = yRef->data.F32[i];
+	    psPlaneTransformApply (&obsCoord, map, &refCoord);
+	    fprintf (f, "%f %f  %f %f  %f %f\n", xObs->data.F32[i], yObs->data.F32[i], xRef->data.F32[i], yRef->data.F32[i], obsCoord.x, obsCoord.y);
+	}
+	fclose (f);
+    }
+
     psFree (xRef);
     psFree (yRef);
@@ -198,4 +256,9 @@
 	// for successful chips, save the measured offsets in the header
 	if (!badAstrom) continue;
+
+	// XXX for now, let's just fail on the bad chips.  In the future, let's try to recover, but we still need to 
+	// catch the failures relative to the model
+	psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO", PS_META_REPLACE, "number of astrometry stars", 0);
+	continue;
 
 	psLogMsg ("psastro", PS_LOG_INFO, "fixing chip %d, angle: %f, pixel: %f,%f\n",
@@ -256,4 +319,7 @@
     }
 
+    psastroVisualPlotFixChips (input, xObs, yObs);
+    psFree (xObs);
+    psFree (yObs);
     psFree (map);
     psFree (view);
Index: branches/bills_081204/psastro/src/psastroLoadRefstars.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroLoadRefstars.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroLoadRefstars.c	(revision 20890)
@@ -137,4 +137,6 @@
         psastroDumpRefstars (refstars, "refstars.dat");
     }
+
+    psastroVisualPlotRefStars (refstars, recipe);
 
     if (psTraceGetLevel("psastro.plot") > 0) {
Index: branches/bills_081204/psastro/src/psastroLuminosityFunction.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroLuminosityFunction.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroLuminosityFunction.c	(revision 20890)
@@ -129,7 +129,5 @@
     lumFunc->sPeak = sPeak;
 
-#if 0
-    psastroLuminosityFunctionPlot(lnMag, Mag, lumFunc, rawFunc);
-#endif
+    psastroVisualPlotLuminosityFunction(lnMag, Mag, lumFunc, rawFunc);
 
     psFree (lnMag);
Index: branches/bills_081204/psastro/src/psastroMaskUpdates.Mosaic.c
===================================================================
--- branches/bills_081204/psastro/src/psastroMaskUpdates.Mosaic.c	(revision 20890)
+++ branches/bills_081204/psastro/src/psastroMaskUpdates.Mosaic.c	(revision 20890)
@@ -0,0 +1,459 @@
+# include "psastroInternal.h"
+
+# define ESCAPE { \
+  psError(PS_ERR_UNKNOWN, false, "I/O failure in psastroMaskUpdate"); \
+  psFree (view); \
+  return false; \
+}
+  
+pmCell *pmCellInChip (pmChip *chip, float x, float y);
+bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip);
+bool psastroMaskCircle (psImage *mask, char value, float x0, float y0, float dX, float dY);
+bool psastroMaskBox (psImage *mask, char value, float x0, float y0, float dL, float dW, float theta);
+void psastroMaskLine (psImage *mask, char value, double x1, double y1, double x2, double y2, int dW);
+void psastroMaskLineBresen (psImage *mask, char value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords);
+void psastroMaskRectangle (psImage *mask, char value, int x0, int y0, int x1, int y1);
+
+// create a mask or mask regions based on the collection of reference stars that are 
+// in the vicinity of each chip
+bool psastroMaskUpdates (pmConfig *config) {
+
+    bool status;
+    pmChip *chip = NULL;
+    pmCell *cell = NULL;
+    pmReadout *readout = NULL;
+    float zeropt, exptime;
+
+    psMaskType maskValue  = pmConfigMaskGet("GHOST", config); // Mask value for ghost pixels
+    psMaskType maskBlank  = pmConfigMaskGet("BLANK", config); // Mask value for blank pixels
+
+    // select the current recipe
+    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, PSASTRO_RECIPE);
+    if (!recipe) {
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find PSASTRO recipe");
+        return false;
+    }
+
+    // this step is optional
+    bool REFSTAR_MASK                      = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK");
+    if (!REFSTAR_MASK) return true;
+
+    double REFSTAR_MASK_MAX_MAG            = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_MAX_MAG");
+    double REFSTAR_MASK_SATSTAR_MAG_SLOPE  = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_MAG_SLOPE");
+    double REFSTAR_MASK_SATSTAR_MAG_MAX    = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_MAG_MAX");
+    double REFSTAR_MASK_SATSTAR_POS_ZERO   = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_POS_ZERO");
+    double REFSTAR_MASK_SATSPIKE_MAG_SLOPE = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_MAG_SLOPE");
+    double REFSTAR_MASK_SATSPIKE_MAG_MAX   = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_MAG_MAX");
+    double REFSTAR_MASK_SATSPIKE_WIDTH     = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_WIDTH");
+
+    // select the input data sources
+    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
+    if (!input) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find input data");
+	return false;
+    }
+    pmFPA *fpa = input->fpa;
+
+    // really error-out here?  or just skip?
+    if (!psastroZeroPointFromRecipe (&zeropt, &exptime, fpa, recipe)) {
+	psLogMsg ("psastro", PS_LOG_INFO, "failed to load zeropt data from recipe");
+	return false;
+    }
+
+    // recipe values are given in instrumental magnitudes
+    // use the zero point and exposure time to convert to apparent mags: M_ap = M_inst + C_0 + 2.5*log(exptime)
+    float MagOffset = zeropt + 2.5*log10(exptime);
+    REFSTAR_MASK_MAX_MAG += MagOffset;
+    REFSTAR_MASK_SATSTAR_MAG_MAX += MagOffset;
+    REFSTAR_MASK_SATSPIKE_MAG_MAX += MagOffset;
+
+    // select the input mask image :: we modify those pixels, the mosaic to chip mosaic format
+    pmFPAfile *inMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT.MASK");
+    if (!inMask) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find input mask");
+	return false;
+    }
+    pmFPA *fpaMask = inMask->fpa;
+
+    // select the output mask image :: we mosaic to chip mosaic format
+    pmFPAfile *outMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.OUTPUT.MASK");
+    if (!outMask) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find output mask");
+	return false;
+    }
+
+    // select the reference mask fpa :: we use this to determine cell boundaries
+    pmFPAfile *refMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.REFMASK");
+    if (!refMask) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find mask reference");
+	return false;
+    }
+
+    double POSANGLE = PM_RAD_DEG * psMetadataLookupF64 (&status, fpa->concepts, "FPA.POSANGLE"); 
+    psAssert (status, "POSANGLE missing");
+
+    // de-activate all files except PSASTRO.INPUT.MASK and PSASTRO.OUTPUT.MASK
+    pmFPAfileActivate (config->files, false, NULL);
+    pmFPAfileActivate (config->files, true, "PSASTRO.INPUT.MASK");
+    pmFPAfileActivate (config->files, true, "PSASTRO.OUTPUT.MASK");
+
+    pmFPAview *view = pmFPAviewAlloc (0);
+    pmFPAview *viewMask = pmFPAviewAlloc (0);
+
+    // open/load files as needed
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+
+    // this loop selects the matched stars for all chips
+    while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) {
+        psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        if (!chip->process || !chip->file_exists) { continue; }
+	if (!chip->fromFPA) { continue; }
+
+	if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+
+	char *filename = NULL;
+	char *chipname = psMetadataLookupStr (&status, chip->concepts, "CHIP.NAME");
+	psStringAppend (&filename, "refstars.mask.%s.dat", chipname);
+	FILE *f = fopen (filename, "w");
+	if (!f) {
+	    psWarning ("cannot create refstar mask file %s\n", filename);
+	    continue;
+	}
+	psFree (filename);
+
+	pmChip *chipMask = pmFPAviewThisChip (view, fpaMask);
+
+	// load sequence for mask corresponding to this chip (XXX this is needed if the input mask is not the same format as the astrometry file
+	*viewMask = *view;
+        while ((cell = pmFPAviewNextCell (viewMask, fpaMask, 1)) != NULL) {
+            psTrace ("psastro", 4, "Mask Cell %d: %x %x\n", viewMask->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+	    if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE;
+	    
+            while ((readout = pmFPAviewNextReadout (viewMask, fpaMask, 1)) != NULL) {
+		if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE;
+                if (! readout->data_exists) { continue; }
+	    }
+	}
+
+        while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+
+            // process each of the readouts
+            // XXX there can only be one readout per chip in astrometry, right?
+            while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
+
+                // select the raw objects for this readout
+                psArray *refstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.REFSTARS");
+                if (refstars == NULL) { continue; }
+
+		// we need to generate the following masks regions:
+		// 1) circle around the saturated stars (scaled by magnitude)
+		// 2) diffraction spikes in direction ROT - ROTo
+		// 3) bleed trail in the direction of the readout
+
+		// XXX for the moment, let's just generate mana region-file objects in chip pixel space
+		for (int i = 0; i < refstars->n; i++) {
+		    pmAstromObj *ref = refstars->data[i];
+		    if (ref->Mag > REFSTAR_MASK_MAX_MAG) continue;
+		    
+		    // convert x,y chip coordinates to cells in maskChip
+		    pmCell *maskCell = pmCellInChip (chipMask, ref->chip->x, ref->chip->y);
+
+		    // XXX convert ref->Mag to instrumental mags
+
+		    // CIRCLE around the stars (scaled by magnitude)
+		    float radius = REFSTAR_MASK_SATSTAR_MAG_SLOPE * (REFSTAR_MASK_SATSTAR_MAG_MAX - ref->Mag);
+		    fprintf (f, "CIRCLE %f %f  %f %f\n", ref->chip->x, ref->chip->y, radius, radius);
+
+		    if (maskCell) {
+			float xCell = 0.0;
+			float yCell = 0.0;
+			pmCellCoordsForChip (&xCell, &yCell, maskCell, ref->chip->x, ref->chip->y);
+			// XXX for now, assume cell binning is 1x1 relative to chip
+			if (maskCell->readouts->n) {
+			    pmReadout *readout = maskCell->readouts->data[0];
+			    psastroMaskCircle (readout->mask, maskValue, xCell, yCell, radius, radius);
+			}
+		    }
+
+		    // LINE for boundaries of the saturation spikes (scaled by magnitude)
+		    float spikeLength = REFSTAR_MASK_SATSPIKE_MAG_SLOPE * (REFSTAR_MASK_SATSPIKE_MAG_MAX - ref->Mag);
+		    float spikeWidth = REFSTAR_MASK_SATSPIKE_WIDTH;
+
+		    for (float theta = 0.0; theta < 2*M_PI; theta += M_PI / 2.0) {
+			float x0, y0, x1, y1, dx, dy;
+
+			float Theta = POSANGLE - REFSTAR_MASK_SATSTAR_POS_ZERO + theta;
+
+			// lower side 
+			x0 = ref->chip->x + spikeWidth*sin(Theta);
+			y0 = ref->chip->y - spikeWidth*cos(Theta);
+			x1 = ref->chip->x + spikeLength*cos(Theta) + spikeWidth*sin(Theta);
+			y1 = ref->chip->y + spikeLength*sin(Theta) - spikeWidth*cos(Theta);
+			dx = x1 - x0;
+			dy = y1 - y0;
+			fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+
+			// upper side 
+			x0 = ref->chip->x - spikeWidth*sin(Theta);
+			y0 = ref->chip->y + spikeWidth*cos(Theta);
+			x1 = ref->chip->x + spikeLength*cos(Theta) - spikeWidth*sin(Theta);
+			y1 = ref->chip->y + spikeLength*sin(Theta) + spikeWidth*cos(Theta);
+			dx = x1 - x0;
+			dy = y1 - y0;
+			fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+
+			if (maskCell) {
+			    float xCell = 0.0;
+			    float yCell = 0.0;
+			    pmCellCoordsForChip (&xCell, &yCell, maskCell, ref->chip->x, ref->chip->y);
+			    int xParityCell = psMetadataLookupS32(NULL, maskCell->concepts, "CELL.XPARITY");
+			    int yParityCell = psMetadataLookupS32(NULL, maskCell->concepts, "CELL.YPARITY");
+			    // XXX for now, assume cell binning is 1x1 relative to chip
+			    // XXX for now, assume only a single readout
+			    if (maskCell->readouts->n) {
+				pmReadout *readout = maskCell->readouts->data[0];
+				psastroMaskBox (readout->mask, maskValue, xCell, yCell, spikeLength, spikeWidth, Theta*xParityCell*yParityCell);
+			    }
+			}
+		    }
+
+		    // LINE for boundaries of the bleed lines
+		    fprintf (f, "LINE %f %f  %f %f\n", ref->chip->x, ref->chip->y, 0.0, -100.0);
+		    if (maskCell) {
+			float xCell = 0.0;
+			float yCell = 0.0;
+			pmCellCoordsForChip (&xCell, &yCell, maskCell, ref->chip->x, ref->chip->y);
+			// XXX for now, assume cell binning is 1x1 relative to chip
+			if (maskCell->readouts->n) {
+			    pmReadout *readout = maskCell->readouts->data[0];
+			    psastroMaskRectangle (readout->mask, maskValue, (int) xCell-3, (int) yCell, (int) xCell+4, readout->mask->numRows);
+			    // XXX this can be done more easily knowing the we mask to the end in y only
+			}
+		    }
+		}
+            }
+        }
+
+	pmChip *outChip = pmFPAviewThisChip(view, outMask->fpa);
+	if (!outChip->hdu && !outChip->parent->hdu) {
+	    const char *name = psMetadataLookupStr(&status, inMask->fpa->concepts, "FPA.OBS"); // Name of FPA
+	    pmFPAAddSourceFromView(outMask->fpa, name, view, outMask->format);
+	}
+
+	psTrace("psastro", 5, "mosaic chip %s to %s (xbin,ybin: %d,%d to %d,%d)\n",
+		inMask->name, outMask->name, inMask->xBin, inMask->yBin, outMask->xBin, outMask->yBin);
+
+	// Mosaic the chip, making a deep copy.  This has the side effect of making the output
+	// image products pure trimmed images, but also increases the memory footprint.
+	status = pmChipMosaic(outChip, chipMask, true, maskBlank);
+
+	// output sequence for mask corresponding to this chip
+	*viewMask = *view;
+        while ((cell = pmFPAviewNextCell (viewMask, outMask->fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Mask Cell %d: %x %x\n", viewMask->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+	    
+            while ((readout = pmFPAviewNextReadout (viewMask, outMask->fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
+		if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_AFTER)) ESCAPE;
+	    }
+	    if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_AFTER)) ESCAPE;
+	}
+
+	if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+	fclose (f);
+    }
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+
+    // deactivate all files
+    pmFPAfileActivate (config->files, false, NULL);
+
+    psFree (view);
+    psFree (viewMask);
+    return true;
+}
+
+// XXX this is going to be very slow...
+pmCell *pmCellInChip (pmChip *chip, float x, float y) {
+
+# if (0)
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell 
+    float xCell = PM_CHIP_TO_CELL(xChip, x0Cell, xParityCell, xBin);
+    float yCell = PM_CHIP_TO_CELL(yChip, y0Cell, yParityCell, yBin);
+# endif
+
+    for (int i = 0; i < chip->cells->n; i++) {
+
+	pmCell *cell = chip->cells->data[i];
+	psRegion *region = pmCellExtent (cell);
+
+	if (x < region->x0) goto skip;
+	if (x > region->x1) goto skip;
+	if (y < region->y0) goto skip;
+	if (y > region->y1) goto skip;
+
+	psFree (region);
+	return cell;
+
+    skip:
+	psFree (region);
+    }
+    return NULL;
+}
+
+// convert chip coords to cell coords, given known cell
+bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip) {
+
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell 
+    // ((pos)*(binning)*(cellParity) + (cell0))
+    // XXX this is probably totally wrong now....
+    // ((pos) - (cell0))*(cellParity)/(binning))
+    *xCell = (xChip - x0Cell)*xParityCell/xBin;
+    *yCell = (yChip - y0Cell)*yParityCell/yBin;
+
+    return true;
+}
+
+// XXX should be doing an OR
+bool psastroMaskCircle (psImage *mask, char value, float x0, float y0, float dX, float dY) {
+
+    // XXX need to worry about row0, col0
+    for (int ix = -dX; ix <= +dX; ix++) {
+	int jx = ix + x0;
+	if (jx < 0) continue;
+	if (jx >= mask->numCols) continue;
+	for (int iy = -dY; iy <= +dY; iy++) {
+	    int jy = iy + y0;
+	    if (jy < 0) continue;
+	    if (jy >= mask->numRows) continue;
+
+	    double r2 = PS_SQR(ix/dX) + PS_SQR(iy/dY);
+	    if (r2 > 1.0) continue;
+	    
+	    mask->data.U8[jy][jx] |= value;
+	}
+    }
+    return true;
+}
+
+// XXX should be doing an OR
+bool psastroMaskBox (psImage *mask, char value, float x0, float y0, float dL, float dW, float theta) {
+
+    // draw a series of lines (from -0.5*dW to +0.5*dW) of length dL, starting at x0, y0, angle theta
+
+    float xs = x0;
+    float ys = y0;
+
+    float xe = xs + dL*cos(theta);
+    float ye = ys + dL*sin(theta);
+
+    psastroMaskLine (mask, value, xs, ys, xe, ye, (int) dW);
+    return true;
+}
+
+// identify the quadrant and draw the correct line
+void psastroMaskLine (psImage *mask, char value, double x1, double y1, double x2, double y2, int dW) {
+
+  int FlipDirect, FlipCoords;
+  int X1, Y1, X2, Y2, dX, dY;
+
+  /* rather than draw the line from float positions, we find the closest
+     integer end-points and draw the line between those pixels */ 
+
+  X1 = ROUND(x1);
+  Y1 = ROUND(y1);
+  X2 = ROUND(x2);
+  Y2 = ROUND(y2);
+
+  dX = X2 - X1;
+  dY = Y2 - Y1;
+
+  FlipCoords = (abs(dX) < abs(dY));
+  FlipDirect = FlipCoords ? (y1 > y2) : (x1 > x2);
+
+  if (!FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X1, Y1, X2, Y2, dW, FALSE);
+  if ( FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X2, Y2, X1, Y1, dW, FALSE);
+  if (!FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y1, X1, Y2, X2, dW, TRUE);
+  if ( FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y2, X2, Y1, X1, dW, TRUE);
+
+  return;
+}
+
+// use the Bresenham line drawing technique
+// integer-only Bresenham line-draw version which is fast
+void psastroMaskLineBresen (psImage *mask, char value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords) {
+
+    int X, Y, dX, dY;
+    int e, e2;
+
+    dX = X2 - X1;
+    dY = Y2 - Y1;
+
+    Y = Y1;
+    e = 0;
+    for (X = X1; X <= X2; X++) {
+	if (X > 0) {
+	    if (swapcoords) {
+		if (X >= mask->numRows) continue;
+		for (int y = Y - dW; y <= Y + dW; y++) {
+		    if (y < 0) continue;
+		    if (y >= mask->numCols) continue;
+		    mask->data.U8[X][y] |= value;
+		}
+	    } else {
+		if (X >= mask->numCols) continue;
+		for (int y = Y - dW; y <= Y + dW; y++) {
+		    if (y < 0) continue;
+		    if (y >= mask->numRows) continue;
+		    mask->data.U8[y][X] |= value;
+		}
+	    }
+	}
+	e += dY;
+	e2 = 2 * e;
+	if (e2 > dX) {
+	    Y++;
+	    e -= dX;
+	} 
+	if (e2 < -dX) {
+	    Y--;
+	    e += dX;
+	}
+    }
+    return;
+}
+
+void psastroMaskRectangle (psImage *mask, char value, int x0, int y0, int x1, int y1) {
+    for (int iy = PS_MAX(0,y0); iy < PS_MIN(y1,mask->numRows); iy++) {
+	for (int ix = PS_MAX(0,x0); ix < PS_MIN(x1,mask->numCols); ix++) {
+	    mask->data.U8[iy][ix] |= value;
+	}
+    }
+}
+
Index: branches/bills_081204/psastro/src/psastroMaskUpdates.c
===================================================================
--- branches/bills_081204/psastro/src/psastroMaskUpdates.c	(revision 20890)
+++ branches/bills_081204/psastro/src/psastroMaskUpdates.c	(revision 20890)
@@ -0,0 +1,493 @@
+# include "psastroInternal.h"
+
+# define ESCAPE { \
+  psError(PS_ERR_UNKNOWN, false, "I/O failure in psastroMaskUpdate"); \
+  psFree (view); \
+  return false; \
+}
+  
+pmCell *pmCellInChip (pmChip *chip, float x, float y);
+bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip);
+bool pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell);
+
+bool psastroMaskCircle (psImage *mask, char value, float x0, float y0, float dX, float dY);
+bool psastroMaskBox (psImage *mask, char value, float x0, float y0, float dL, float dW, float theta);
+void psastroMaskLine (psImage *mask, char value, double x1, double y1, double x2, double y2, int dW);
+void psastroMaskLineBresen (psImage *mask, char value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords);
+void psastroMaskRectangle (psImage *mask, char value, int x0, int y0, int x1, int y1);
+
+// create a mask or mask regions based on the collection of reference stars that are 
+// in the vicinity of each chip
+bool psastroMaskUpdates (pmConfig *config) {
+
+    bool status;
+    pmChip *chip = NULL;
+    pmCell *cell = NULL;
+    pmReadout *readout = NULL;
+    float zeropt, exptime;
+
+    psMaskType maskValue  = pmConfigMaskGet("GHOST", config); // Mask value for ghost pixels
+
+    // psMaskType maskBlank  = pmConfigMaskGet("BLANK", config); // Mask value for blank pixels
+
+    // select the current recipe
+    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, PSASTRO_RECIPE);
+    if (!recipe) {
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find PSASTRO recipe");
+        return false;
+    }
+
+    // this step is optional
+    bool REFSTAR_MASK                      = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK");
+    if (!REFSTAR_MASK) return true;
+
+    bool REFSTAR_MASK_REGIONS              = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_REGIONS");
+
+    double REFSTAR_MASK_MAX_MAG            = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_MAX_MAG");
+    double REFSTAR_MASK_SATSTAR_MAG_SLOPE  = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_MAG_SLOPE");
+    double REFSTAR_MASK_SATSTAR_MAG_MAX    = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_MAG_MAX");
+    double REFSTAR_MASK_SATSTAR_POS_ZERO   = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSTAR_POS_ZERO");
+    double REFSTAR_MASK_SATSPIKE_MAG_SLOPE = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_MAG_SLOPE");
+    double REFSTAR_MASK_SATSPIKE_MAG_MAX   = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_MAG_MAX");
+    double REFSTAR_MASK_SATSPIKE_WIDTH     = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_SATSPIKE_WIDTH");
+    double REFSTAR_MASK_BLEED_MAG_MAX      = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_BLEED_MAG_MAX");
+    double REFSTAR_MASK_BLEED_MAG_SLOPE    = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_BLEED_MAG_SLOPE");
+
+    // select the input data sources
+    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
+    if (!input) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find input data");
+	return false;
+    }
+    pmFPA *fpa = input->fpa;
+
+    // really error-out here?  or just skip?
+    if (!psastroZeroPointFromRecipe (&zeropt, &exptime, fpa, recipe)) {
+	psLogMsg ("psastro", PS_LOG_INFO, "failed to load zeropt data from recipe");
+	return false;
+    }
+
+    // recipe values are given in instrumental magnitudes
+    // use the zero point and exposure time to convert to apparent mags: M_ap = M_inst + C_0 + 2.5*log(exptime)
+    float MagOffset = zeropt + 2.5*log10(exptime);
+    REFSTAR_MASK_MAX_MAG += MagOffset;
+    REFSTAR_MASK_SATSTAR_MAG_MAX += MagOffset;
+    REFSTAR_MASK_SATSPIKE_MAG_MAX += MagOffset;
+    REFSTAR_MASK_BLEED_MAG_MAX += MagOffset;
+
+    // select the output mask image :: we mosaic to chip mosaic format
+    pmFPAfile *outMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.OUTPUT.MASK");
+    if (!outMask) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find output mask");
+	return false;
+    }
+    pmFPA *fpaMask = outMask->fpa;
+
+    // select the reference mask fpa :: we use this to determine cell boundaries
+    pmFPAfile *refMask = psMetadataLookupPtr (NULL, config->files, "PSASTRO.REFMASK");
+    if (!refMask) {
+	psError(PSASTRO_ERR_CONFIG, true, "Can't find mask reference");
+	return false;
+    }
+
+    double POSANGLE = PM_RAD_DEG * psMetadataLookupF64 (&status, fpa->concepts, "FPA.POSANGLE"); 
+    psAssert (status, "POSANGLE missing");
+
+    // de-activate all files except PSASTRO.INPUT.MASK and PSASTRO.OUTPUT.MASK
+    pmFPAfileActivate (config->files, false, NULL);
+    pmFPAfileActivate (config->files, true, "PSASTRO.INPUT.MASK");
+    pmFPAfileActivate (config->files, true, "PSASTRO.OUTPUT.MASK");
+    pmFPAfileActivate (config->files, true, "PSASTRO.REFMASK");
+
+    pmFPAview *view = pmFPAviewAlloc (0);
+    pmFPAview *viewMask = pmFPAviewAlloc (0);
+
+    // open/load files as needed
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+
+    // this loop selects the matched stars for all chips
+    while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) {
+        psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        if (!chip->process || !chip->file_exists) { continue; }
+	if (!chip->fromFPA) { continue; }
+
+	if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+
+	// text region files for testing
+	FILE *f = NULL;
+	if (REFSTAR_MASK_REGIONS) {
+	  char *filename = NULL;
+	  char *chipname = psMetadataLookupStr (&status, chip->concepts, "CHIP.NAME");
+	  psStringAppend (&filename, "refstars.mask.%s.dat", chipname);
+	  FILE *f = fopen (filename, "w");
+	  if (!f) {
+	    psWarning ("cannot create refstar mask file %s\n", filename);
+	    continue;
+	  }
+	  psFree (filename);
+	}
+
+	pmChip *refChip  = pmFPAviewThisChip (view, refMask->fpa);
+
+	// load sequence for mask corresponding to this chip (XXX this is needed if the input mask is not the same format as the astrometry file
+	*viewMask = *view;
+        while ((cell = pmFPAviewNextCell (viewMask, fpaMask, 1)) != NULL) {
+            psTrace ("psastro", 4, "Mask Cell %d: %x %x\n", viewMask->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+	    if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE;
+	    
+            while ((readout = pmFPAviewNextReadout (viewMask, fpaMask, 1)) != NULL) {
+		if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_BEFORE)) ESCAPE;
+                if (! readout->data_exists) { continue; }
+	    }
+	}
+
+        while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+
+	    // the input mask is a chip-mosaic image
+	    // the output mask is a chip-mosaic image
+	    // we mark the masked pixels in the chip space, BUT
+	    // we need to find the ends of the cells for the bleeds
+
+	    // we mask pixels on the input mask image (chip-mosaic)
+	    pmCell *cellMask = pmFPAviewThisCell(view, outMask->fpa);
+	    pmReadout *readoutMask = NULL;
+	    if (cellMask->readouts->n) {
+		readoutMask = cellMask->readouts->data[0];
+	    }
+
+            // process each of the readouts
+            // XXX there can only be one readout per chip in astrometry, right?
+            while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
+
+                // select the raw objects for this readout
+                psArray *refstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.REFSTARS");
+                if (refstars == NULL) { continue; }
+
+		// we need to generate the following masks regions:
+		// 1) circle around the saturated stars (scaled by magnitude)
+		// 2) diffraction spikes in direction ROT - ROTo
+		// 3) bleed trail in the direction of the readout
+
+		// XXX for the moment, let's just generate mana region-file objects in chip pixel space
+		for (int i = 0; i < refstars->n; i++) {
+		    pmAstromObj *ref = refstars->data[i];
+		    if (ref->Mag > REFSTAR_MASK_MAX_MAG) continue;
+		    
+		    // XXX convert ref->Mag to instrumental mags
+
+		    // CIRCLE around the stars (scaled by magnitude)
+		    float radius = REFSTAR_MASK_SATSTAR_MAG_SLOPE * (REFSTAR_MASK_SATSTAR_MAG_MAX - ref->Mag);
+
+		    if (REFSTAR_MASK_REGIONS) {
+		      fprintf (f, "CIRCLE %f %f  %f %f\n", ref->chip->x, ref->chip->y, radius, radius);
+		    }
+
+		    // XXX for now, assume cell binning is 1x1 relative to chip
+		    if (readoutMask) {
+			psastroMaskCircle (readoutMask->mask, maskValue, ref->chip->x, ref->chip->y, radius, radius);
+		    }
+
+		    // LINE for boundaries of the saturation spikes (scaled by magnitude)
+		    float spikeLength = REFSTAR_MASK_SATSPIKE_MAG_SLOPE * (REFSTAR_MASK_SATSPIKE_MAG_MAX - ref->Mag);
+		    float spikeWidth = REFSTAR_MASK_SATSPIKE_WIDTH;
+
+		    for (float theta = 0.0; theta < 2*M_PI; theta += M_PI / 2.0) {
+			float x0, y0, x1, y1, dx, dy;
+
+			float Theta = POSANGLE - REFSTAR_MASK_SATSTAR_POS_ZERO + theta;
+
+			if (REFSTAR_MASK_REGIONS) {
+			  // lower side 
+			  x0 = ref->chip->x + spikeWidth*sin(Theta);
+			  y0 = ref->chip->y - spikeWidth*cos(Theta);
+			  x1 = ref->chip->x + spikeLength*cos(Theta) + spikeWidth*sin(Theta);
+			  y1 = ref->chip->y + spikeLength*sin(Theta) - spikeWidth*cos(Theta);
+			  dx = x1 - x0;
+			  dy = y1 - y0;
+
+			  fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+
+			  // upper side 
+			  x0 = ref->chip->x - spikeWidth*sin(Theta);
+			  y0 = ref->chip->y + spikeWidth*cos(Theta);
+			  x1 = ref->chip->x + spikeLength*cos(Theta) - spikeWidth*sin(Theta);
+			  y1 = ref->chip->y + spikeLength*sin(Theta) + spikeWidth*cos(Theta);
+			  dx = x1 - x0;
+			  dy = y1 - y0;
+			  fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+			}
+
+			if (readoutMask) {
+			    psastroMaskBox (readoutMask->mask, maskValue, ref->chip->x, ref->chip->y, spikeLength, spikeWidth, Theta);
+			}
+		    }
+
+		    // convert x,y chip coordinates to cells in maskChip
+		    pmCell *refCell = pmCellInChip (refChip, ref->chip->x, ref->chip->y);
+
+		    // LINE for boundaries of the bleed lines
+		    if (REFSTAR_MASK_REGIONS) {
+		      fprintf (f, "LINE %f %f  %f %f\n", ref->chip->x, ref->chip->y, 0.0, -100.0);
+		    }
+
+		    if (readoutMask && refCell) {
+			float xCell = 0.0;
+			float yCell = 0.0;
+			float xEnd = 0.0;
+			float yEnd = 0.0;
+			// find coordinate of star on cell
+			pmCellCoordsForChip (&xCell, &yCell, refCell, ref->chip->x, ref->chip->y);
+			// find coordinate of end-point on chip
+
+			int ySize = psMetadataLookupS32(NULL, refCell->concepts, "CELL.YSIZE");
+			pmChipCoordsForCell (&xEnd, &yEnd, refCell, xCell, ySize);
+
+			float width = REFSTAR_MASK_BLEED_MAG_SLOPE*(REFSTAR_MASK_BLEED_MAG_MAX - ref->Mag);
+			psastroMaskRectangle (readoutMask->mask, maskValue, (int) ref->chip->x-0.5*width, (int) ref->chip->y, (int) ref->chip->x+0.5*width+1, yEnd);
+		    }
+		}
+            }
+        }
+
+	// output sequence for mask corresponding to this chip
+	*viewMask = *view;
+        while ((cell = pmFPAviewNextCell (viewMask, outMask->fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Mask Cell %d: %x %x\n", viewMask->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+	    
+            while ((readout = pmFPAviewNextReadout (viewMask, outMask->fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
+		if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_AFTER)) ESCAPE;
+	    }
+	    if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_AFTER)) ESCAPE;
+	}
+
+	if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+	if (REFSTAR_MASK_REGIONS) {
+	  fclose (f);
+	}
+    }
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+
+    // deactivate all files
+    pmFPAfileActivate (config->files, false, NULL);
+
+    psFree (view);
+    psFree (viewMask);
+    return true;
+}
+
+// XXX this is going to be very slow...
+pmCell *pmCellInChip (pmChip *chip, float x, float y) {
+
+# if (0)
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell 
+    float xCell = PM_CHIP_TO_CELL(xChip, x0Cell, xParityCell, xBin);
+    float yCell = PM_CHIP_TO_CELL(yChip, y0Cell, yParityCell, yBin);
+# endif
+
+    for (int i = 0; i < chip->cells->n; i++) {
+
+	pmCell *cell = chip->cells->data[i];
+	psRegion *region = pmCellExtent (cell);
+
+	if (x < region->x0) goto skip;
+	if (x > region->x1) goto skip;
+	if (y < region->y0) goto skip;
+	if (y > region->y1) goto skip;
+
+	psFree (region);
+	return cell;
+
+    skip:
+	psFree (region);
+    }
+    return NULL;
+}
+
+// convert chip coords to cell coords, given known cell
+bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip) {
+
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell 
+    // ((pos)*(binning)*(cellParity) + (cell0))
+    // XXX this is probably totally wrong now....
+    // ((pos) - (cell0))*(cellParity)/(binning))
+    *xCell = (xChip - x0Cell)*xParityCell/xBin;
+    *yCell = (yChip - y0Cell)*yParityCell/yBin;
+
+    return true;
+}
+
+// convert chip coords to cell coords, given known cell
+bool pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell) {
+
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell 
+    // ((pos)*(binning)*(cellParity) + (cell0))
+    // XXX this is probably totally wrong now....
+    // ((pos) - (cell0))*(cellParity)/(binning))
+    *xChip = xCell*xBin*xParityCell + x0Cell; 
+    *yChip = yCell*yBin*yParityCell + y0Cell; 
+
+    return true;
+}
+
+// XXX should be doing an OR
+bool psastroMaskCircle (psImage *mask, char value, float x0, float y0, float dX, float dY) {
+
+    // XXX need to worry about row0, col0
+    for (int ix = -dX; ix <= +dX; ix++) {
+	int jx = ix + x0;
+	if (jx < 0) continue;
+	if (jx >= mask->numCols) continue;
+	for (int iy = -dY; iy <= +dY; iy++) {
+	    int jy = iy + y0;
+	    if (jy < 0) continue;
+	    if (jy >= mask->numRows) continue;
+
+	    double r2 = PS_SQR(ix/dX) + PS_SQR(iy/dY);
+	    if (r2 > 1.0) continue;
+	    
+	    mask->data.U8[jy][jx] |= value;
+	}
+    }
+    return true;
+}
+
+// XXX should be doing an OR
+bool psastroMaskBox (psImage *mask, char value, float x0, float y0, float dL, float dW, float theta) {
+
+    // draw a series of lines (from -0.5*dW to +0.5*dW) of length dL, starting at x0, y0, angle theta
+
+    float xs = x0;
+    float ys = y0;
+
+    float xe = xs + dL*cos(theta);
+    float ye = ys + dL*sin(theta);
+
+    psastroMaskLine (mask, value, xs, ys, xe, ye, (int) dW);
+    return true;
+}
+
+// identify the quadrant and draw the correct line
+void psastroMaskLine (psImage *mask, char value, double x1, double y1, double x2, double y2, int dW) {
+
+  int FlipDirect, FlipCoords;
+  int X1, Y1, X2, Y2, dX, dY;
+
+  /* rather than draw the line from float positions, we find the closest
+     integer end-points and draw the line between those pixels */ 
+
+  X1 = ROUND(x1);
+  Y1 = ROUND(y1);
+  X2 = ROUND(x2);
+  Y2 = ROUND(y2);
+
+  dX = X2 - X1;
+  dY = Y2 - Y1;
+
+  FlipCoords = (abs(dX) < abs(dY));
+  FlipDirect = FlipCoords ? (y1 > y2) : (x1 > x2);
+
+  if (!FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X1, Y1, X2, Y2, dW, FALSE);
+  if ( FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X2, Y2, X1, Y1, dW, FALSE);
+  if (!FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y1, X1, Y2, X2, dW, TRUE);
+  if ( FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y2, X2, Y1, X1, dW, TRUE);
+
+  return;
+}
+
+// use the Bresenham line drawing technique
+// integer-only Bresenham line-draw version which is fast
+void psastroMaskLineBresen (psImage *mask, char value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords) {
+
+    int X, Y, dX, dY;
+    int e, e2;
+
+    dX = X2 - X1;
+    dY = Y2 - Y1;
+
+    Y = Y1;
+    e = 0;
+    for (X = X1; X <= X2; X++) {
+	if (X > 0) {
+	    if (swapcoords) {
+		if (X >= mask->numRows) continue;
+		for (int y = Y - dW; y <= Y + dW; y++) {
+		    if (y < 0) continue;
+		    if (y >= mask->numCols) continue;
+		    mask->data.U8[X][y] |= value;
+		}
+	    } else {
+		if (X >= mask->numCols) continue;
+		for (int y = Y - dW; y <= Y + dW; y++) {
+		    if (y < 0) continue;
+		    if (y >= mask->numRows) continue;
+		    mask->data.U8[y][X] |= value;
+		}
+	    }
+	}
+	e += dY;
+	e2 = 2 * e;
+	if (e2 > dX) {
+	    Y++;
+	    e -= dX;
+	} 
+	if (e2 < -dX) {
+	    Y--;
+	    e += dX;
+	}
+    }
+    return;
+}
+
+void psastroMaskRectangle (psImage *mask, char value, int x0, int y0, int x1, int y1) {
+    
+    int xs = PS_MAX (0, PS_MIN (mask->numCols, PS_MIN (x0, x1)));
+    int xe = PS_MAX (0, PS_MIN (mask->numCols, PS_MAX (x0, x1)));
+    int ys = PS_MAX (0, PS_MIN (mask->numRows, PS_MIN (y0, y1)));
+    int ye = PS_MAX (0, PS_MIN (mask->numRows, PS_MAX (y0, y1)));
+
+    for (int iy = ys; iy < ye; iy++) {
+	for (int ix = xs; ix < xe; ix++) {
+	    mask->data.U8[iy][ix] |= value;
+	}
+    }
+}
+
Index: branches/bills_081204/psastro/src/psastroMosaicAstrom.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroMosaicAstrom.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroMosaicAstrom.c	(revision 20890)
@@ -13,6 +13,6 @@
     psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, PSASTRO_RECIPE);
     if (!recipe) {
-	psError(PSASTRO_ERR_CONFIG, false, "Can't find PSASTRO recipe!\n");
-	return false;
+        psError(PSASTRO_ERR_CONFIG, false, "Can't find PSASTRO recipe!\n");
+        return false;
     }
 
@@ -20,6 +20,6 @@
     pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
     if (!input) {
-	psError(PSASTRO_ERR_CONFIG, false, "Can't find input data!\n");
-	return false;
+        psError(PSASTRO_ERR_CONFIG, false, "Can't find input data!\n");
+        return false;
     }
 
@@ -27,13 +27,13 @@
 
     // before we do object matches, we need to (optionally) fix failed chips.  We compare chips with
-    // the supplied mosaic model.  Adjust significant outliers to match model.  
+    // the supplied mosaic model.  Adjust significant outliers to match model.
     # if (0)
     if (!psastroFixChips (config, recipe)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to align problematic chips");
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to align problematic chips");
+        return false;
     }
     if (!psastroFixChipsTest (config, recipe)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to align problematic chips");
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to align problematic chips");
+        return false;
     }
     # endif
@@ -50,14 +50,14 @@
     // first, re-perform the match with a slightly tighter circle
     if (!psastroMosaicSetMatch (fpa, recipe, 4)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to match raw and ref stars for mosaic (4th pass)");
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to match raw and ref stars for mosaic (4th pass)");
+        return false;
     }
     if (!psastroMosaicChipAstrom (fpa, recipe, 4)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure chip astrometry in mosaic mode (4th pass)");
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure chip astrometry in mosaic mode (4th pass)");
+        return false;
     }
-    if (psTraceGetLevel("psastro.dump") > 0) { 
-	snprintf (filename, 256, "%s.10.dat", outroot);
-	psastroDumpMatches (fpa, filename); 
+    if (psTraceGetLevel("psastro.dump") > 0) {
+        snprintf (filename, 256, "%s.10.dat", outroot);
+        psastroDumpMatches (fpa, filename);
     }
 
@@ -66,10 +66,10 @@
     psMetadata *updates = psMetadataLookupMetadata (&status, fpa->analysis, "PSASTRO.HEADER");
     if (!updates) {
-	updates = psMetadataAlloc ();
-	psMetadataAddMetadata (fpa->analysis, PS_LIST_TAIL, "PSASTRO.HEADER",  PS_META_REPLACE, "psastro header stats", updates);
-	psFree (updates);
+        updates = psMetadataAlloc ();
+        psMetadataAddMetadata (fpa->analysis, PS_LIST_TAIL, "PSASTRO.HEADER",  PS_META_REPLACE, "psastro header stats", updates);
+        psFree (updates);
     }
     if (!pmAstromWriteBilevelMosaic (updates, fpa, NONLIN_TOL)) {
-	psAbort ("failed to save header terms");
+        psAbort ("failed to save header terms");
     }
 
@@ -103,11 +103,11 @@
     // is this needed? yes, if we didn't do SingleChip astrometry first
     if (!psastroMosaicSetMatch (fpa, recipe, pass)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to match raw and ref stars for mosaic (pass %d)", pass);
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to match raw and ref stars for mosaic (pass %d)", pass);
+        return false;
     }
 
-    if ((pass == 0) && (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1)) { 
-	snprintf (filename, 256, "%s.0.dat", rootname);
-	psastroDumpMatches (fpa, filename); 
+    if ((pass == 0) && (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1)) {
+        snprintf (filename, 256, "%s.0.dat", rootname);
+        psastroDumpMatches (fpa, filename);
     }
 
@@ -116,11 +116,11 @@
     // then recalculate raw and ref positions
     if (!psastroMosaicCommonScale (fpa, recipe)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to set a common scale for the chips (pass %d)", pass);
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to set a common scale for the chips (pass %d)", pass);
+        return false;
     }
 
-    if ((pass == 0) && (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1)) { 
-	snprintf (filename, 256, "%s.1.dat", rootname);
-	psastroDumpMatches (fpa, filename); 
+    if ((pass == 0) && (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1)) {
+        snprintf (filename, 256, "%s.1.dat", rootname);
+        psastroDumpMatches (fpa, filename);
     }
 
@@ -129,24 +129,20 @@
     // refit the per-chip terms with linear fits only
     if (!psastroMosaicDistortion (fpa, recipe, pass)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure mosaic gradients (pass %d)", pass);
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure mosaic gradients (pass %d)", pass);
+        return false;
     }
 
     snprintf (filename, 256, "%s.%d.dat", rootname, 2*pass + 2);
     if (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1) { psastroDumpMatches (fpa, filename); }
-    
+
     // measure the astrometry for the chips under the distortion term
     if (!psastroMosaicChipAstrom (fpa, recipe, pass)) {
-	psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure chip astrometry in mosaic mode (pass %d)", pass);
-	return false;
+        psError(PSASTRO_ERR_UNKNOWN, false, "failed to measure chip astrometry in mosaic mode (pass %d)", pass);
+        return false;
     }
 
-    int traceLevel = psTraceGetLevel("psastro.dump.psastroMosaicAstrom");
-    if (traceLevel > 1) {
-	snprintf (filename, 256, "%s.%d.dat", rootname, 2*pass + 3);
-	psastroDumpMatches (fpa, filename); 
-    } else {
-	snprintf (filename, 256, "%s.dat", rootname);
-	psastroDumpMatches (fpa, filename); 
+    if (psTraceGetLevel("psastro.dump.psastroMosaicAstrom") > 1) {
+        snprintf (filename, 256, "%s.%d.dat", rootname, 2*pass + 3);
+        psastroDumpMatches (fpa, filename);
     }
 
Index: branches/bills_081204/psastro/src/psastroMosaicChipAstrom.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroMosaicChipAstrom.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroMosaicChipAstrom.c	(revision 20890)
@@ -36,12 +36,18 @@
                 if (!psastroMosaicOneChip (chip, readout, recipe, updates, iteration)) {
                     readout->data_exists = false;
-                    psError(PS_ERR_UNKNOWN, false, "failed to find a solution for %d,%d,%d\n",
-                            view->chip, view->cell, view->readout);
-                    psFree(view);
-                    return false;
+                    psError(PS_ERR_UNKNOWN, false, "failed to find a solution for %d,%d,%d\n", view->chip, view->cell, view->readout);
+		    psErrorStackPrint(stderr, "failure for one chip\n");
+		    psErrorClear();
+		    continue;
                 }
 
                 // create the header keywords to descripe the results
-                pmAstromWriteBilevelChip (updates, chip, NONLIN_TOL);
+                if (!pmAstromWriteBilevelChip (updates, chip, NONLIN_TOL)) {
+                    readout->data_exists = false;
+                    psError(PS_ERR_UNKNOWN, false, "invalid solution for %d,%d,%d\n", view->chip, view->cell, view->readout);
+		    psErrorStackPrint(stderr, "failure for one chip\n");
+		    psErrorClear();
+		    continue;
+                }
             }
         }
Index: branches/bills_081204/psastro/src/psastroMosaicOneChip.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroMosaicOneChip.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroMosaicOneChip.c	(revision 20890)
@@ -5,5 +5,5 @@
   if (!status) { \
    psError(PSASTRO_ERR_CONFIG, false, MESSAGE); \
-   return false; } 
+   return false; }
 
 bool psastroMosaicOneChip (pmChip *chip, pmReadout *readout, psMetadata *recipe, psMetadata *updates, int iteration) {
@@ -29,5 +29,5 @@
 
     // correct radius to FP units (physical pixel scale in microns per pixel)
-    REQUIRED_RECIPE_VALUE (double pixelScale, "PSASTRO.PIXEL.SCALE", F32, "Failed to lookup pixel scale"); 
+    REQUIRED_RECIPE_VALUE (double pixelScale, "PSASTRO.PIXEL.SCALE", F32, "Failed to lookup pixel scale");
 
     // allowed limits for valid solutions
@@ -42,15 +42,15 @@
     int order = psMetadataLookupS32 (&status, recipe, orderWord);
     if (!status || (order == -1)) {
-	order = defaultOrder;
+        order = defaultOrder;
     }
 
     // modify the order to correspond to the actual number of matched stars:
-    if ((match->n < 15) && (order >= 3)) order = 2;
-    if ((match->n < 11) && (order >= 2)) order = 1;
-    if ((match->n <  8) && (order >= 1)) order = 0;
+    if ((match->n < 17) && (order >= 3)) order = 2;
+    if ((match->n < 13) && (order >= 2)) order = 1;
+    if ((match->n <  9) && (order >= 1)) order = 0;
     if ((match->n <  3) || (order < 0) || (order > 3)) {
-	psLogMsg ("psastro", 3, "insufficient stars (%ld) or invalid order (%d)", match->n, order); 
-	return false; 
-    } 
+        psLogMsg ("psastro", 3, "insufficient stars (%ld) or invalid order (%d)", match->n, order);
+        return false;
+    }
 
     psLogMsg ("psastro", PS_LOG_DETAIL, "mosaic fit chip order %d", order);
@@ -60,25 +60,25 @@
     // coefficients frozen to the current values
     if (order == 0) {
-	// set FIT mask for all higher order terms of the existing solution
-	// any existing SET masks will be retained.
-	for (int i = 0; i <= chip->toFPA->x->nX; i++) {
-	    for (int j = 0; j <= chip->toFPA->x->nY; j++) {
-		if (i + j > 0) {
-		    chip->toFPA->x->coeffMask[i][j] |= PS_POLY_MASK_FIT;
-		    chip->toFPA->y->coeffMask[i][j] |= PS_POLY_MASK_FIT;
-		}
-	    }
-	}
+        // set FIT mask for all higher order terms of the existing solution
+        // any existing SET masks will be retained.
+        for (int i = 0; i <= chip->toFPA->x->nX; i++) {
+            for (int j = 0; j <= chip->toFPA->x->nY; j++) {
+                if (i + j > 0) {
+                    chip->toFPA->x->coeffMask[i][j] |= PS_POLY_MASK_FIT;
+                    chip->toFPA->y->coeffMask[i][j] |= PS_POLY_MASK_FIT;
+                }
+            }
+        }
     } else {
-	psFree (chip->toFPA);
-	chip->toFPA = psPlaneTransformAlloc (order, order);
-	for (int i = 0; i <= chip->toFPA->x->nX; i++) {
-	    for (int j = 0; j <= chip->toFPA->x->nY; j++) {
-		if (i + j > order) {
-		    chip->toFPA->x->coeffMask[i][j] = PS_POLY_MASK_SET;
-		    chip->toFPA->y->coeffMask[i][j] = PS_POLY_MASK_SET;
-		}
-	    }
-	}
+        psFree (chip->toFPA);
+        chip->toFPA = psPlaneTransformAlloc (order, order);
+        for (int i = 0; i <= chip->toFPA->x->nX; i++) {
+            for (int j = 0; j <= chip->toFPA->x->nY; j++) {
+                if (i + j > order) {
+                    chip->toFPA->x->coeffMask[i][j] = PS_POLY_MASK_SET;
+                    chip->toFPA->y->coeffMask[i][j] = PS_POLY_MASK_SET;
+                }
+            }
+        }
     }
 
@@ -87,11 +87,11 @@
     psStats *fitStats = NULL;
 //    if (order == 0) {
-//	fitStats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
-//	fitStats->clipSigma = psMetadataLookupF32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NSIGMA");
-//	fitStats->clipIter = psMetadataLookupS32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NITER");
+//      fitStats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
+//      fitStats->clipSigma = psMetadataLookupF32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NSIGMA");
+//      fitStats->clipIter = psMetadataLookupS32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NITER");
 //    } else {
-	fitStats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
-	fitStats->clipSigma = psMetadataLookupF32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NSIGMA");
-	fitStats->clipIter = psMetadataLookupS32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NITER");
+        fitStats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
+        fitStats->clipSigma = psMetadataLookupF32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NSIGMA");
+        fitStats->clipIter = psMetadataLookupS32 (&status, recipe, "PSASTRO.MOSAIC.CHIP.NITER");
 //    }
 
@@ -99,6 +99,6 @@
     pmAstromFitResults *results = pmAstromMatchFit (chip->toFPA, rawstars, refstars, match, fitStats);
     if (!results) {
-	psError(PSASTRO_ERR_DATA, false, "failed to perform the matched fit\n");
-	return false;
+        psError(PSASTRO_ERR_DATA, false, "failed to perform the matched fit\n");
+        return false;
     }
 
@@ -124,10 +124,10 @@
     psLogMsg ("psastro", PS_LOG_INFO, "astrometry solution: error: %f arcsec, Nstars: %d", astError, astNstar);
     if ((maxError > 0) && (astError > maxError)) {
-	psLogMsg("psastro", PS_LOG_INFO, "residual error is too large, failed to find a solution: %f > %f", astError, maxError);
-	validSolution = false;
+        psLogMsg("psastro", PS_LOG_INFO, "residual error is too large, failed to find a solution: %f > %f", astError, maxError);
+        validSolution = false;
     }
     if (astNstar < minNstar) {
-	psLogMsg("psastro", PS_LOG_INFO, "solution uses too few stars: %d < %d", astNstar, minNstar);
-	validSolution = false;
+        psLogMsg("psastro", PS_LOG_INFO, "solution uses too few stars: %d < %d", astNstar, minNstar);
+        validSolution = false;
     }
 
@@ -136,9 +136,9 @@
     psMetadataAddF32 (updates, PS_LIST_TAIL, "CERROR",   PS_META_REPLACE, "astrometry error (arcsec)", astError);
     if (validSolution) {
-	psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", astError/sqrt(astNstar));
-	psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO",   PS_META_REPLACE, "number of astrometry stars", astNstar);
+        psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", astError/sqrt(astNstar));
+        psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO",   PS_META_REPLACE, "number of astrometry stars", astNstar);
     } else {
-	psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", 0.0);
-	psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO",   PS_META_REPLACE, "number of astrometry stars", 0);
+        psMetadataAddF32 (updates, PS_LIST_TAIL, "CPRECISE", PS_META_REPLACE, "astrometry precision (arcsec)", 0.0);
+        psMetadataAddS32 (updates, PS_LIST_TAIL, "NASTRO",   PS_META_REPLACE, "number of astrometry stars", 0);
     }
     psMetadataAddF32 (updates, PS_LIST_TAIL, "EQUINOX",  PS_META_REPLACE, "", 2000.0); // XXX this is bogus: should be defined based on equinox of refstars
@@ -147,4 +147,7 @@
     psastroUpdateChipToFPA (fpa, chip, rawstars, refstars);
 
+    //plot results
+    psastroVisualPlotMosaicOneChip(rawstars, refstars, match, recipe);
+
     psFree (fitStats);
     psFree (results);
Index: branches/bills_081204/psastro/src/psastroMosaicSetMatch.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroMosaicSetMatch.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroMosaicSetMatch.c	(revision 20890)
@@ -10,22 +10,22 @@
 
     // use small radius to match stars (assume starting astrometry is good)
-    bool status = false; 
+    bool status = false;
     sprintf (radiusWord, "PSASTRO.MOSAIC.RADIUS.N%d", iteration);
-    double RADIUS = psMetadataLookupF32 (&status, recipe, radiusWord); 
-    if (!status) { 
-	psError(PS_ERR_IO, false, "Failed to lookup matching radius: %s", radiusWord); 
-	psFree (view);
-	return false; 
-    } 
+    double RADIUS = psMetadataLookupF32 (&status, recipe, radiusWord);
+    if (!status) {
+        psError(PS_ERR_IO, false, "Failed to lookup matching radius: %s", radiusWord);
+        psFree (view);
+        return false;
+    }
 
     if (RADIUS <= 0.0) {
-	if (iteration == 0) {
-	    psError(PS_ERR_IO, false, "Invalid match radius for first iteration: %s", radiusWord); 
-	    psFree (view);
-	    return false; 
-	} 
-	psWarning ("skipping match for iteration %d\n", iteration);
-	psFree (view);
-	return true;
+        if (iteration == 0) {
+            psError(PS_ERR_IO, false, "Invalid match radius for first iteration: %s", radiusWord);
+            psFree (view);
+            return false;
+        }
+        psWarning ("skipping match for iteration %d\n", iteration);
+        psFree (view);
+        return true;
     }
 
@@ -34,32 +34,34 @@
         psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
         if (!chip->process || !chip->file_exists) { continue; }
-	if (!chip->fromFPA) { continue; }
-	
-	while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
+        if (!chip->fromFPA) { continue; }
+
+        while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
             psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
             if (!cell->process || !cell->file_exists) { continue; }
 
-	    // process each of the readouts
-	    // XXX there can only be one readout per chip, right?
-	    while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
-		if (! readout->data_exists) { continue; }
+            // process each of the readouts
+            // XXX there can only be one readout per chip, right?
+            while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
 
-		// select the raw objects for this readout
-		psArray *rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.RAWSTARS");
-		if (rawstars == NULL) { continue; }
+                // select the raw objects for this readout
+                psArray *rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.RAWSTARS");
+                if (rawstars == NULL) { continue; }
 
-		// select the raw objects for this readout
-		psArray *refstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.REFSTARS");
-		if (refstars == NULL) { continue; }
-		psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n);
+                // select the raw objects for this readout
+                psArray *refstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.REFSTARS");
+                if (refstars == NULL) { continue; }
+                psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n);
 
-		psArray *matches = pmAstromRadiusMatchChip (rawstars, refstars, RADIUS);
-		psTrace ("psastro", 4, "Matched %ld refstars\n", matches->n);
+                psArray *matches = pmAstromRadiusMatchChip (rawstars, refstars, RADIUS);
+                psTrace ("psastro", 4, "Matched %ld refstars\n", matches->n);
 
-		// XXX drop the old one
-		psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.MATCH", PS_DATA_ARRAY | PS_META_REPLACE, "astrometry matches", matches);
-		psFree (matches);
-	    }
-	}
+                psastroVisualPlotMosaicMatches(rawstars, refstars, matches, iteration, recipe);
+
+                // XXX drop the old one
+                psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.MATCH", PS_DATA_ARRAY | PS_META_REPLACE, "astrometry matches", matches);
+                psFree (matches);
+            }
+        }
     }
     psFree (view);
Index: branches/bills_081204/psastro/src/psastroOneChipFit.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroOneChipFit.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroOneChipFit.c	(revision 20890)
@@ -51,7 +51,12 @@
 
 	// modify the order to correspond to the actual number of matched stars:
-	if ((match->n < 11) && (order >= 3)) order = 2;
-	if ((match->n <  7) && (order >= 2)) order = 1;
-	if ((match->n <  4) && (order >= 1)) order = 0;
+	int Ndof_min = 3;
+	int order_max = 0.5*(3 + sqrt(4*match->n - 4*Ndof_min + 1));
+	order = PS_MIN (order, order_max);
+
+	// if ((match->n < 11) && (order >= 3)) order = 2;
+	// if ((match->n <  7) && (order >= 2)) order = 1;
+	// if ((match->n <  4) && (order >= 1)) order = 0;
+
 	if (order < 1) {
 	    psLogMsg ("psastro", 3, "insufficient stars or invalid order: %ld stars", match->n); 
@@ -151,4 +156,6 @@
     }
 
+    psastroVisualPlotOneChipFit (rawstars, refstars, match, recipe);
+
     if (psTraceGetLevel("psastro.plot") > 0) {
 	psastroPlotOneChipFit (rawstars, refstars, match, recipe);
Index: branches/bills_081204/psastro/src/psastroRemoveClumps.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroRemoveClumps.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroRemoveClumps.c	(revision 20890)
@@ -11,11 +11,11 @@
     float Ymax = obj->FP->y;
     for (int i = 0; i < input->n; i++) {
-	obj = (pmAstromObj *)input->data[i];
-	if (!isfinite(obj->FP->x)) continue;
-	if (!isfinite(obj->FP->y)) continue;
-	Xmin = PS_MIN (Xmin, obj->FP->x);
-	Xmax = PS_MAX (Xmax, obj->FP->x);
-	Ymin = PS_MIN (Ymin, obj->FP->y);
-	Ymax = PS_MAX (Ymax, obj->FP->y);
+        obj = (pmAstromObj *)input->data[i];
+        if (!isfinite(obj->FP->x)) continue;
+        if (!isfinite(obj->FP->y)) continue;
+        Xmin = PS_MIN (Xmin, obj->FP->x);
+        Xmax = PS_MAX (Xmax, obj->FP->x);
+        Ymin = PS_MIN (Ymin, obj->FP->y);
+        Ymax = PS_MAX (Ymax, obj->FP->y);
     }
 
@@ -27,10 +27,10 @@
     // accumulate 2D histogram in image
     for (int i = 0; i < input->n; i++) {
-	obj = (pmAstromObj *)input->data[i];
-	if (!isfinite(obj->FP->x)) continue;
-	if (!isfinite(obj->FP->y)) continue;
-	int Xi = PS_MIN (PS_MAX((obj->FP->x - Xmin) / scale + 5, 0), count->numCols);
-	int Yi = PS_MIN (PS_MAX((obj->FP->y - Ymin) / scale + 5, 0), count->numRows);
-	count->data.U32[Yi][Xi] ++;
+        obj = (pmAstromObj *)input->data[i];
+        if (!isfinite(obj->FP->x)) continue;
+        if (!isfinite(obj->FP->y)) continue;
+        int Xi = PS_MIN (PS_MAX((obj->FP->x - Xmin) / scale + 5, 0), count->numCols);
+        int Yi = PS_MIN (PS_MAX((obj->FP->y - Ymin) / scale + 5, 0), count->numRows);
+        count->data.U32[Yi][Xi] ++;
     }
 
@@ -38,15 +38,15 @@
     psStats *stats = psStatsAlloc (PS_STAT_MAX | PS_STAT_MAX | PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
     if (!psImageStats(stats, count, NULL, 0)) {
-	psError(PS_ERR_UNKNOWN, false, "Unable to get image statistics.\n");
-	psFree(stats);
-	psFree(count);
-	return NULL;
+        psError(PS_ERR_UNKNOWN, false, "Unable to get image statistics.\n");
+        psFree(stats);
+        psFree(count);
+        return NULL;
     }
 
     if (stats->max < 1) {
-	psError(PS_ERR_UNKNOWN, false, "no valid sources in image\n");
-	psFree(stats);
-	psFree(count);
-	return NULL;
+        psError(PS_ERR_UNKNOWN, false, "no valid sources in image\n");
+        psFree(stats);
+        psFree(count);
+        return NULL;
     }
 
@@ -55,14 +55,16 @@
     psTrace ("psastro", 4, "skipping stars in cells with more than %f stars\n", limit);
 
+    psastroVisualPlotRemoveClumps (input, count, scale, limit);
+
     // find and exclude objects in bad pixels
     psArray *output = psArrayAllocEmpty (input->n);
     for (int i = 0; i < input->n; i++) {
-	obj = (pmAstromObj *)input->data[i];
-	if (!isfinite(obj->FP->x)) continue;
-	if (!isfinite(obj->FP->y)) continue;
-	int Xi = PS_MIN (PS_MAX((obj->FP->x - Xmin) / scale + 5, 0), count->numCols);
-	int Yi = PS_MIN (PS_MAX((obj->FP->y - Ymin) / scale + 5, 0), count->numRows);
-	if (count->data.U32[Yi][Xi] > limit) continue;
-	psArrayAdd (output, 16, obj);
+        obj = (pmAstromObj *)input->data[i];
+        if (!isfinite(obj->FP->x)) continue;
+        if (!isfinite(obj->FP->y)) continue;
+        int Xi = PS_MIN (PS_MAX((obj->FP->x - Xmin) / scale + 5, 0), count->numCols);
+        int Yi = PS_MIN (PS_MAX((obj->FP->y - Ymin) / scale + 5, 0), count->numRows);
+        if (count->data.U32[Yi][Xi] > limit) continue;
+        psArrayAdd (output, 16, obj);
     }
 
@@ -77,11 +79,11 @@
 for (int iy = 0; iy < count->numRows; iy++) {
     for (int ix = 0; ix < count->numCols; ix++) {
-	if (count->data.U32[iy][ix] > limit) {
-	    psPlane *pixel = psPlaneAlloc();
-	    pixel->x = ix;
-	    pixel->y = iy;
-	    psArrayAdd (badpix, 16, pixel);
-	    psFree (pixel);
-	}
+        if (count->data.U32[iy][ix] > limit) {
+            psPlane *pixel = psPlaneAlloc();
+            pixel->x = ix;
+            pixel->y = iy;
+            psArrayAdd (badpix, 16, pixel);
+            psFree (pixel);
+        }
     }
 }
Index: branches/bills_081204/psastro/src/psastroUtils.c
===================================================================
--- branches/cnb_branch_20081104/psastro/src/psastroUtils.c	(revision 20530)
+++ branches/bills_081204/psastro/src/psastroUtils.c	(revision 20890)
@@ -20,60 +20,75 @@
 
     float pixelScaleUse = 1.0, pixelScale1 = 1.0,  pixelScale2 = 1.0,  pixelScale = 1.0;
+    psVector *oldScale = psVectorAllocEmpty (fpa->chips->n, PS_TYPE_F32);
 
     char *option = psMetadataLookupStr (NULL, recipe, "PSASTRO.COMMON.SCALE.OPTION");
     if (option == NULL) {
-	psError(PSASTRO_ERR_DATA, false, "no choice set for common scale option\n");
-	return false;
+        psError(PSASTRO_ERR_DATA, false, "no choice set for common scale option\n");
+        return false;
     }
 
     bool useExternal = true;
+    int nobj = 0;
 
     // find the min or max scale chip
     if (!strcasecmp (option, "MIN") || !strcasecmp (option, "MAX")) {
 
-	bool useMax = !strcasecmp (option, "MAX");
-	pixelScaleUse = (useMax) ? FLT_MIN : FLT_MAX;
-
-	for (int i = 0; i < fpa->chips->n; i++) {
-	    pmChip *chip = fpa->chips->data[i];
-	    if (!chip->process || !chip->file_exists) { continue; }
-	    if (!chip->toFPA) { continue; }
-
-	    pixelScale1 = hypot (chip->toFPA->x->coeff[1][0], chip->toFPA->x->coeff[0][1]);
-	    pixelScale2 = hypot (chip->toFPA->y->coeff[1][0], chip->toFPA->y->coeff[0][1]);
-	    pixelScale = 0.5*(pixelScale1 + pixelScale2);
-	    
-	    pixelScaleUse = (useMax) ? PS_MAX (pixelScale, pixelScaleUse) : PS_MIN (pixelScale, pixelScaleUse);
-	}
-	useExternal = false;
-    }
+        bool useMax = !strcasecmp (option, "MAX");
+        pixelScaleUse = (useMax) ? FLT_MIN : FLT_MAX;
+
+        for (int i = 0; i < fpa->chips->n; i++) {
+            pmChip *chip = fpa->chips->data[i];
+            if (!chip->process || !chip->file_exists) { continue; }
+            if (!chip->toFPA) { continue; }
+
+	    if (chip->cells->n == 0) { continue; }
+	    pmCell *cell = chip->cells->data[0];
+            if (!cell->process || !cell->file_exists) { continue; }
+
+	    if (cell->readouts->n == 0) { continue; }
+	    pmReadout *readout = cell->readouts->data[0];
+	    if (! readout->data_exists) { continue; }
+
+            pixelScale1 = hypot (chip->toFPA->x->coeff[1][0], chip->toFPA->x->coeff[0][1]);
+            pixelScale2 = hypot (chip->toFPA->y->coeff[1][0], chip->toFPA->y->coeff[0][1]);
+            pixelScale = 0.5*(pixelScale1 + pixelScale2);
+            oldScale->data.F32[nobj++] = pixelScale;
+            pixelScaleUse = (useMax) ? PS_MAX (pixelScale, pixelScaleUse) : PS_MIN (pixelScale, pixelScaleUse);
+        }
+        useExternal = false;
+        oldScale->n = nobj;
+    }
+
 
     // rescale each chip by the reference scale
     for (int i = 0; i < fpa->chips->n; i++) {
-	pmChip *chip = fpa->chips->data[i];
-	if (!chip->process || !chip->file_exists) { continue; }
-	if (!chip->toFPA) { continue; }
-
-	psPlaneTransform *toFPA = chip->toFPA;
-	psPlaneTransform *fromFPA = chip->fromFPA;
-	    
-	pixelScale1 = hypot (toFPA->x->coeff[1][0], toFPA->x->coeff[0][1]);
-	pixelScale2 = hypot (toFPA->y->coeff[1][0], toFPA->y->coeff[0][1]);
-
-	if (useExternal) {
-	    pixelScaleUse = getChipPixelScale (chip);
-	}
-
-	for (int i = 0; i <= toFPA->x->nX; i++) {
-	    for (int j = 0; j <= toFPA->x->nX; j++) {
-		toFPA->x->coeff[i][j] *= pixelScaleUse/pixelScale1;
-		toFPA->y->coeff[i][j] *= pixelScaleUse/pixelScale2;
-		fromFPA->x->coeff[i][j] *= pixelScale1/pixelScaleUse;
-		fromFPA->y->coeff[i][j] *= pixelScale2/pixelScaleUse;
-	    }
-	}
-
+        pmChip *chip = fpa->chips->data[i];
+        if (!chip->process || !chip->file_exists) { continue; }
+        if (!chip->toFPA) { continue; }
+
+        psPlaneTransform *toFPA = chip->toFPA;
+        psPlaneTransform *fromFPA = chip->fromFPA;
+
+        pixelScale1 = hypot (toFPA->x->coeff[1][0], toFPA->x->coeff[0][1]);
+        pixelScale2 = hypot (toFPA->y->coeff[1][0], toFPA->y->coeff[0][1]);
+
+        if (useExternal) {
+            pixelScaleUse = getChipPixelScale (chip);
+        }
+
+        for (int i = 0; i <= toFPA->x->nX; i++) {
+            for (int j = 0; j <= toFPA->x->nX; j++) {
+                toFPA->x->coeff[i][j] *= pixelScaleUse/pixelScale1;
+                toFPA->y->coeff[i][j] *= pixelScaleUse/pixelScale2;
+                fromFPA->x->coeff[i][j] *= pixelScale1/pixelScaleUse;
+                fromFPA->y->coeff[i][j] *= pixelScale2/pixelScaleUse;
+            }
+        }
     }
     psastroMosaicSetAstrom (fpa);
+    if (!useExternal) {
+        psastroVisualPlotCommonScale (fpa, oldScale);
+    }
+    psFree (oldScale);
     return true;
 }
Index: branches/bills_081204/psastro/src/psastroVisual.c
===================================================================
--- branches/bills_081204/psastro/src/psastroVisual.c	(revision 20890)
+++ branches/bills_081204/psastro/src/psastroVisual.c	(revision 20890)
@@ -0,0 +1,1303 @@
+/***********************************/
+/***Diagnostic plots for psastro****/
+/***********************************/
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+# include "psastroInternal.h"
+# include <string.h>
+# include <stdlib.h>
+# include <stdio.h>
+
+# if (HAVE_KAPA)
+# include <kapa.h>
+
+# define KAPAX  700
+# define KAPAY  700
+
+//variables to determine when things are plotted
+static bool isVisual             = false;
+static bool plotRawStars         = true;
+static bool plotRefStars         = false;
+static bool plotLumFunc          = true;
+static bool plotRemoveClumps     = true;
+static bool plotOneChipFit       = true;
+static bool plotFixChips         = true;
+static bool plotAstromGuessCheck = true;
+static bool plotMosaicMatches    = true;
+static bool plotCommonScale      = true;
+static bool plotMosaicOneChip    = true;
+
+// variables to store plotting window indices
+static int kapa = -1;
+static int kapa2 = -1;
+
+// helper routine prototypes
+bool psastroVisualInitGraph (int kapa, KapaSection *section, Graphdata *graphdata);
+bool psastroVisualTriplePlot (int kapa, Graphdata *graphdata, psVector *xVec, psVector *yVec, psVector *zVec, bool increasing);
+bool psastroVisualScaleGraphdata(Graphdata *graphdata, psVector *xVec, psVector *yVec, bool clip);
+bool psastroVisualTripleOverplot (int kapa, Graphdata *graphdata, psVector *xVec, psVector *yVec, psVector *zVec, bool increasing);
+bool psastroVisualCreateScaleVec (psVector *zVec, psVector *zScale, bool increasing);
+bool psastroVisualResidPlot (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe, char *title);
+
+
+/*****************************/
+/** Initialization Routines **/
+/*****************************/
+
+/**  start or stop plotting */
+bool psastroSetVisual (bool mode) {
+    isVisual = mode;
+    return true;
+}
+
+
+/** open, name, and resize a window if necessary*/
+bool psastroVisualInitWindow( int *kapid, char *name ) {
+    if (*kapid == -1) {
+        *kapid = KapaOpenNamedSocket("kapa", name);
+        if (*kapid == -1) {
+            fprintf (stderr, "failure to open kapa; visual mode disabled\n");
+            isVisual = false;
+            return false;
+        }
+        KapaResize (*kapid, KAPAX, KAPAY);
+    }
+    return true;
+}
+
+
+/** ask the user how to proceed*/
+bool psastroVisualAskUser( bool *plotflag ) {
+    char key[10];
+    fprintf (stdout, "[c]ontinue? [s]kip the rest of these plots? [a]bort all visual plots?");
+    if (!fgets(key, 8, stdin)) {
+        psWarning("Unable to read option");
+    }
+    if (key[0] == 's') {
+        *plotflag = false;
+    }
+    if (key[0] == 'a') {
+        psastroSetVisual(false);
+        pmAstromSetVisual(false);
+    }
+    return true;
+}
+
+
+/** destroy windows at the end of a run*/
+bool psastroVisualClose() {
+    if(kapa != -1)
+        KiiClose(kapa);
+    if (kapa2 != -1)
+        KiiClose(kapa2);
+    return true;
+}
+
+
+/***********************/
+/** Plotting Routines **/
+/***********************/
+
+
+/**
+ * Plot raw stars as determined from first pass astrometry fit
+ * Called within psastroAstromGeuss
+ */
+bool psastroVisualPlotRawStars (psArray *rawstars, pmFPA *fpa, pmChip *chip, psMetadata *recipe)
+{
+    // make sure we want to plot this
+    if (!plotRawStars || !isVisual) return true;
+
+    //set up plot region
+    if (!psastroVisualInitWindow (&kapa, "psastro:plots"))
+        return false;
+    Graphdata graphdata;
+    KapaSection section;
+
+    KapaInitGraph (&graphdata);
+    KapaClearPlots (kapa);
+
+    graphdata.color = KapaColorByName ("black");
+    graphdata.ptype = 7;
+    graphdata.size = 0.5;
+    graphdata.style = 2;
+
+    section.dx = 0.4;
+    section.dy = 0.4;
+
+    //initialize and populate plotting vectors
+    bool status = false;
+    float iMagMin = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.INST.MAG.MIN");
+    float iMagMax = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.INST.MAG.MAX");
+
+    psVector *xVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+    psVector *yVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+    psVector *zVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+
+    section.x = 0.0;
+    section.y = 0.5;
+    section.name = NULL;
+    psStringAppend (&section.name, "a0");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    //Chip coordinates
+    int n = 0;
+    for (int i = 0; i < rawstars->n; i++) {
+        pmAstromObj *raw = rawstars->data[i];
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->x;
+        yVec->data.F32[n] = raw->chip->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel(kapa, "Chip", KAPA_LABEL_XP);
+    KapaSendLabel(kapa, "X", KAPA_LABEL_XM);
+    KapaSendLabel(kapa, "Y", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    //Focal Plane Coordinates
+    section.x = 0.5;
+    section.y = 0.5;
+    section.name = NULL;
+    psStringAppend (&section.name, "a1");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < rawstars->n; i++) {
+        pmAstromObj *raw = rawstars->data[i];
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+
+        xVec->data.F32[n] = raw->FP->x;
+        yVec->data.F32[n] = raw->FP->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "Focal Plane", KAPA_LABEL_XP);
+    KapaSendLabel (kapa, "L", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "M", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // Tangent Plane Coordinates
+    section.x = 0.0;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a2");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < rawstars->n; i++) {
+        pmAstromObj *raw = rawstars->data[i];
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+
+        xVec->data.F32[n] = raw->TP->x;
+        yVec->data.F32[n] = raw->TP->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "Tangential Plane", KAPA_LABEL_XP);
+    KapaSendLabel (kapa, "P", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Q", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    //sky coordinates
+    section.x = 0.5;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a3");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < rawstars->n; i++) {
+        pmAstromObj *raw = rawstars->data[i];
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+
+        xVec->data.F32[n] = DEG_RAD*raw->sky->r;
+        yVec->data.F32[n] = DEG_RAD*raw->sky->d;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "Sky", KAPA_LABEL_XP);
+    KapaSendLabel(kapa, "RA", KAPA_LABEL_XM);
+    KapaSendLabel(kapa, "Dec", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // flip x (East increase to left)
+    SWAP (graphdata.xmin, graphdata.xmax);
+    KapaSetLimits (kapa, &graphdata);
+
+    // plot label
+    section.x = 0.0;
+    section.y = 0.0;
+    section.dx = .95;
+    section.dy = .95;
+    section.name = NULL;
+    psStringAppend (&section.name, "a5");
+    KapaSetSection (kapa, &section);
+    KapaSendLabel (kapa, "Raw Star Selection - Initial Astrometry", KAPA_LABEL_XP);
+    psFree (section.name);
+
+    // pause and wait for user input:
+    psastroVisualAskUser( &plotRawStars );
+
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+    return true;
+}
+
+
+/**
+ * plot the location of references stars over the entire fpa
+ * invoked during psastroChooseRefStars
+ */
+bool psastroVisualPlotRefStars (psArray *refstars, psMetadata *recipe)
+{
+    //make sure we want to plot this
+    if (!isVisual || !plotRefStars) return true;
+
+    //set up plotting variables
+    if (!psastroVisualInitWindow (&kapa, "psastro:plots"))
+        return false;
+
+    Graphdata graphdata;
+    KapaInitGraph (&graphdata);
+    KapaClearSections (kapa);
+
+    graphdata.color = KapaColorByName ("black");
+    graphdata.ptype = 7;
+    graphdata.size = 0.5;
+    graphdata.style = 2;
+
+    //initialize and populate plot vectors
+    bool status = false;
+    float rMagMin = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.REF.MAG.MIN");
+    float rMagMax = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.REF.MAG.MAX");
+
+    psVector *xVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+    psVector *yVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+    psVector *zVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+
+    int n = 0;
+    for (int i = 0; i < refstars->n; i++) {
+        pmAstromObj *ref = refstars->data[i];
+        if (!isfinite(ref->Mag)) continue;
+        if (ref->Mag > rMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+
+        xVec->data.F32[n] = DEG_RAD*ref->sky->r;
+        yVec->data.F32[n] = DEG_RAD*ref->sky->d;
+        zVec->data.F32[n] = ref->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "RA", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Dec", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Reference Stars", KAPA_LABEL_XP);
+    psastroVisualTriplePlot(kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // flip x (East increase to left)
+    SWAP (graphdata.xmin, graphdata.xmax);
+    KapaSetLimits (kapa, &graphdata);
+
+    // pause and wait for user input:
+    psastroVisualAskUser(&plotRefStars);
+
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+    return true;
+}
+
+
+/**
+ * Plot the two luminosity functions created within psastroRefStarSubset
+ * The luminosity functions are used to select a subset of reference stars,
+ * so we plot the cutoff that defines this subset
+ */
+bool psastroVisualPlotLuminosityFunction (psVector *lnMag,   ///< Log(n) for each magnitude bin
+                                          psVector *Mag,     ///< magnitude bins
+                                          pmLumFunc *lumFunc,///< Fit to the reference star luminosity function
+                                          pmLumFunc *rawFunc ///< Fit to the raw star luminoisty function
+                                          )
+{
+
+    // make sure we want to plot this
+    if ( !isVisual || !plotLumFunc ) return true;
+
+    //set up plotting variables
+    if ( !psastroVisualInitWindow (&kapa, "psastro:plots"))
+        return false;
+
+    Graphdata graphdata;
+    KapaSection section1 = {"s1", .1, .1, .85, .35};
+    KapaSection section2 =  {"s2", .1, .5, .85, .35};
+    KapaSection section;
+    if(rawFunc == NULL)
+        section = section1;
+    else
+        section = section2;
+    if (rawFunc == NULL)
+        KapaClearPlots(kapa);
+    KapaInitGraph(&graphdata);
+
+    //Determine Plot Limits
+    psastroVisualScaleGraphdata(&graphdata, Mag, lnMag, false);
+
+    //Make a line for the fit
+    float x[2] = {graphdata.xmin, graphdata.xmax};
+    float y[2] = {lumFunc->offset + x[0] * lumFunc->slope,
+                 lumFunc->offset + x[1] * lumFunc->slope};
+
+    //Plot Data
+    KapaSetSection(kapa, &section);
+    KapaSetLimits(kapa, &graphdata);
+
+    KapaSetFont (kapa, "helvetica", 14);
+    KapaBox (kapa, &graphdata);
+    KapaSendLabel (kapa, "Magnitude", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Log(N)", KAPA_LABEL_YM);
+    if (rawFunc == NULL)
+        KapaSendLabel (kapa, "Raw Star Luminosity Function", KAPA_LABEL_XP);
+    else
+        KapaSendLabel (kapa,
+                       "Reference Star Luminosity Function, Shifted Raw Fit, and Cutoff",
+                       KAPA_LABEL_XP);
+    graphdata.color=KapaColorByName("black");
+    graphdata.style = 1;
+    KapaPrepPlot (kapa, lnMag->n, &graphdata);
+    KapaPlotVector(kapa, lnMag->n,   Mag->data.F32, "x");
+    KapaPlotVector(kapa, lnMag->n, lnMag->data.F32, "y");
+
+    //Overplot fit
+    graphdata.style=0;
+    KapaPrepPlot(kapa,2,&graphdata);
+    KapaPlotVector(kapa, 2, x, "x");
+    KapaPlotVector(kapa, 2, y, "y");
+
+    //If rawFunc was supplied, overplot the raw star fit + cutoff
+    if( rawFunc != NULL) {
+        double mRef = 0.5*(lumFunc->mMin + lumFunc->mMax);
+        double logRho = mRef * lumFunc->slope + lumFunc->offset;
+        double mRaw = (logRho - rawFunc->offset) / rawFunc->slope;
+        double deltaM = mRef - mRaw;
+        double mRefMax = rawFunc->mMax + deltaM;
+
+        float xraw[2] = {rawFunc->mMin + deltaM, rawFunc->mMax + deltaM};
+        float yraw[2] = {rawFunc->offset + (rawFunc->slope) * rawFunc->mMin,
+                        rawFunc->offset + (rawFunc->slope) * rawFunc->mMax};
+        float x[2] = {mRefMax, mRefMax};
+        float y[2] = {graphdata.ymin, graphdata.ymax};
+        graphdata.color= KapaColorByName("red");
+        KapaPrepPlot(kapa, 2, &graphdata);
+        KapaPlotVector(kapa, 2, x, "x");
+        KapaPlotVector(kapa, 2, y, "y");
+        KapaPrepPlot (kapa, 2, &graphdata);
+        KapaPlotVector (kapa, 2, xraw, "x");
+        KapaPlotVector (kapa, 2, yraw, "y");
+
+        // pause and wait for user input:
+        psastroVisualAskUser (&plotLumFunc);
+    }
+    return true;
+} // end of psastroVisualPlotLuminosityFunction
+
+
+/**
+ * Plot the stars in a region, and indicate which stars are part of 'clumps'
+ * These stars are flagged during astrometric fitting, since dense regions are
+ * harder to cross-match than sparse ones. Called during psastroRemoveClumps.
+ */
+bool psastroVisualPlotRemoveClumps (psArray *input, ///< Array containing the field stars
+                                    psImage *count, ///< A 2D histogram of the field star distribution
+                                    int scale,      ///< The pixel size of the histogram
+                                    float limit     ///< The minimum numuber of stars in a bin flagged as a clump
+                                    )
+{
+
+    //make sure we want to plot this
+    if (!isVisual || !plotRemoveClumps) return true;
+
+    //set up plot variables
+    if ( !psastroVisualInitWindow (&kapa, "psastro:plots")) return false;
+
+    KapaSection section;
+    Graphdata graphdata;
+    section.x = 0;
+    section.dx = .9;
+    section.y = 0;
+    section.dy = .9;
+    section.name = NULL;
+    psStringAppend( &section.name, "a0");
+    KapaInitGraph(&graphdata);
+    KapaSetSection(kapa, &section);
+    psFree(section.name);
+
+    graphdata.ptype = 7;
+    graphdata.size = 0.5;
+    graphdata.style = 2;
+    graphdata.color = KapaColorByName ("black");
+    KapaClearPlots(kapa);
+
+    //set up plot vectors
+    float Xmin = +FLT_MAX;
+    float Xmax = -FLT_MAX;
+    float Ymin = +FLT_MAX;
+    float Ymax = -FLT_MAX;
+    psVector *xVec = psVectorAlloc (input->n, PS_TYPE_F32);
+    psVector *yVec = psVectorAlloc (input->n, PS_TYPE_F32);
+
+    //determine boundaries for histogram bin calculation
+    int n = 0;
+    for (int i=0; i< input->n; i++) {
+        pmAstromObj *obj = (pmAstromObj *)input->data[i];
+        if (!isfinite(obj->FP->x)) continue;
+        if (!isfinite(obj->FP->y)) continue;
+        xVec->data.F32[n] = obj->FP->x;
+        yVec->data.F32[n] = obj->FP->y;
+        Xmin = PS_MIN (Xmin, xVec->data.F32[n]);
+        Xmax = PS_MAX (Xmax, xVec->data.F32[n]);
+        Ymin = PS_MIN (Ymin, yVec->data.F32[n]);
+        Ymax = PS_MAX (Ymax, yVec->data.F32[n]);
+        n++;
+    }
+    xVec->n = yVec->n = n;
+
+    //plot stars
+    graphdata.xmax = Xmax;
+    graphdata.xmin = Xmin;
+    graphdata.ymax = Ymax;
+    graphdata.ymin = Ymin;
+    KapaSetLimits (kapa, &graphdata);
+    KapaSetFont (kapa, "helvetica", 14);
+
+    KapaBox (kapa, &graphdata);
+
+    KapaSendLabel (kapa, "L (pixels)", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "M (pixels)", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Regions Flagged as Clumps (Red Boxes)",
+                   KAPA_LABEL_XP);
+
+    KapaPrepPlot (kapa, xVec->n, &graphdata);
+    KapaPlotVector (kapa, xVec->n, xVec->data.F32, "x");
+    KapaPlotVector (kapa, yVec->n, yVec->data.F32, "y");
+
+    graphdata.color = KapaColorByName ("red");
+    graphdata.style = 1;
+
+    //overplot clumpy regions excluded from analysis
+    for(int i = 0; i < count->numCols; i++) {
+        for (int j = 0; j < count->numRows; j++) {
+            if(count->data.U32[j][i] <= limit) continue; //not a clump
+            float Xbot = (i - 5) * scale + Xmin;
+            float Ybot = (j - 5) * scale + Ymin;
+            if(Xbot < graphdata.xmin || Xbot > graphdata.xmax ||
+               Ybot < graphdata.ymin || Ybot > graphdata.ymax) continue;
+            float x[5] = {Xbot, Xbot + scale, Xbot + scale, Xbot, Xbot};
+            float y[5] = {Ybot, Ybot, Ybot + scale, Ybot + scale, Ybot};
+            KapaPrepPlot (kapa, 5, &graphdata);
+            KapaPlotVector (kapa, 5, x, "x");
+            KapaPlotVector (kapa, 5, y, "y");
+        }
+    }
+
+    //ask for user input and finish
+    psastroVisualAskUser (&plotRemoveClumps);
+    psFree (xVec);
+    psFree (yVec);
+
+    return true;
+}
+
+
+/**
+ * Assess the goodness of fit for a signle chip by
+ * plotting the fit residuals
+ * invoked during psastroOneChipFit
+ */
+bool psastroVisualPlotOneChipFit (psArray *rawstars, ///< stars detected in the image
+                                  psArray *refstars, ///< reference stars over the same region
+                                  psArray *match,    ///< contains which rawstars match to which refstars
+                                  psMetadata *recipe ///< data reduction recipe
+                                  )
+{
+
+    //make sure we want to plot this
+    if (!isVisual || !plotOneChipFit)
+        return true;
+
+    //plot the residuals
+    if (!psastroVisualResidPlot(rawstars, refstars, match, recipe, "Single Chip Fit Residuals (Chip Coordinates)"))
+        return false;
+
+    //ask for user input and finish
+    psastroVisualAskUser(&plotOneChipFit);
+    return true;
+}
+
+
+/**
+ * Plots the chip corners in the FP before and after chips with inconsistent solutions have been fixed.
+ * Invoked during psastroFixChips
+ */
+bool psastroVisualPlotFixChips (pmFPAfile *input, ///< focal plane array file
+                                psVector *xOld, ///< old X location of chip cornerss
+                                psVector *yOld ///< old Y location of chip corners
+                                )
+{
+    //make sure we want to plot this
+    if(!isVisual || !plotFixChips) return true;
+
+    if(!psastroVisualInitWindow(&kapa, "psastro:plots")) return false;
+
+    KapaSection section = {"s1", .05, .05, .9, .9};
+    Graphdata graphdata;
+    KapaInitGraph( &graphdata);
+    KapaClearPlots (kapa);
+    graphdata.ptype = 2;
+    graphdata.style = 2;
+
+    psVector *xNew = psVectorAllocEmpty (xOld->n, PS_TYPE_F32);
+    psVector *yNew = psVectorAllocEmpty (yOld->n, PS_TYPE_F32);
+
+    // copy of the code in psastroFixChips that generated xOld, yOld, but for xNew, yNew
+    pmFPAview *view = pmFPAviewAlloc (0);
+
+    pmChip *obsChip = NULL;
+    while ((obsChip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) {
+        if (!obsChip->process || !obsChip->file_exists || !obsChip->data_exists) { continue; }
+
+        psRegion *region = pmChipPixels (obsChip);
+        psPlane ptCP, ptFP;
+
+        ptCP.x = region->x0; ptCP.y = region->y0;
+        psPlaneTransformApply (&ptFP, obsChip->toFPA, &ptCP);
+	psVectorAppend (xNew, ptFP.x);
+	psVectorAppend (yNew, ptFP.y);
+
+        ptCP.x = region->x0; ptCP.y = region->y1;
+        psPlaneTransformApply (&ptFP, obsChip->toFPA, &ptCP);
+	psVectorAppend (xNew, ptFP.x);
+	psVectorAppend (yNew, ptFP.y);
+
+        ptCP.x = region->x1; ptCP.y = region->y1;
+        psPlaneTransformApply (&ptFP, obsChip->toFPA, &ptCP);
+	psVectorAppend (xNew, ptFP.x);
+	psVectorAppend (yNew, ptFP.y);
+
+        ptCP.x = region->x1; ptCP.y = region->y0;
+        psPlaneTransformApply (&ptFP, obsChip->toFPA, &ptCP);
+	psVectorAppend (xNew, ptFP.x);
+	psVectorAppend (yNew, ptFP.y);
+
+        psFree (region);
+    }
+
+    //set up graph
+    psastroVisualScaleGraphdata(&graphdata, xOld, yOld, true);
+    psastroVisualInitGraph(kapa, &section, &graphdata);
+    KapaSendLabel (kapa, "L (FP)", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "M (FP)", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Chip corners before (black) and after (red) FixChips", KAPA_LABEL_XP);
+    KapaPrepPlot (kapa, xOld->n, &graphdata);
+    KapaPlotVector (kapa, xOld->n, xOld->data.F32, "x");
+    KapaPlotVector (kapa, xOld->n, yOld->data.F32, "y");
+
+    graphdata.ptype = 1;
+    graphdata.color = KapaColorByName("red");
+    KapaPrepPlot (kapa, xNew->n, &graphdata);
+    KapaPlotVector (kapa, xNew->n, xNew->data.F32, "x");
+    KapaPlotVector (kapa, xNew->n, yNew->data.F32, "y");
+
+    psastroVisualAskUser(&plotFixChips);
+    psFree(xNew);
+    psFree(yNew);
+    psFree(view);
+    return true;
+}
+
+
+/**
+ *  Plots the fpa chip corners projected on to the tangential plane before and after
+ *  the astrometry solution has been applied. In psastroAstromGuessCheck, the old corners
+ *  are then fit to the new corners to get a sense at how far off the initial WCS info was
+ *  in offset, rotation, and scale. This procedure also plots the residuals of the fit from
+ *  old to new coordinates
+ */
+bool psastroVisualPlotAstromGuessCheck (psVector *cornerPo, ///< P coordinates of chip corners before fitting
+                                        psVector *cornerQo, ///< Q coordinates of chip corners before fitting
+                                        psVector *cornerPn, ///< P coordinates of chip corners after fitting
+                                        psVector *cornerQn, ///< Q coordinates of chip corners after fitting
+                                        psVector *cornerPd, ///< P coordinate residuals of fit from old to new coordinates
+                                        psVector *cornerQd  ///< Q coordinate residuals of fit from old to new coordinates
+                                        )
+{
+
+    //make sure we want to plot this
+    if (!isVisual || !plotAstromGuessCheck) return true;
+
+    //set up graph window
+    if ( !psastroVisualInitWindow (&kapa, "psastro:plots"))
+        return false;
+    Graphdata graphdata;
+    KapaSection section;
+    KapaInitGraph(&graphdata);
+    KapaClearPlots (kapa);
+
+    graphdata.color = KapaColorByName ("black");
+    graphdata.ptype = 2;
+    graphdata.size = 0.5;
+    graphdata.style = 2;
+
+    section.dx = 0.4;
+    section.dy = 0.4;
+
+    //Old Corners
+    section.x = 0.30;
+    section.y = 0.50;
+    section.name = NULL;
+    psStringAppend (&section.name, "a0");
+    KapaSetSection (kapa, &section);
+    psFree(section.name);
+
+    psastroVisualScaleGraphdata (&graphdata, cornerPo, cornerPo, true);
+    KapaSetLimits (kapa, &graphdata);
+    KapaBox (kapa, &graphdata);
+    KapaSendLabel (kapa, "P (Pixels)", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "Q (Pixels)", KAPA_LABEL_YM);
+    KapaSendLabel (kapa,
+                   "Fiducial Points in the Tangent Plane. Black: Initial Astrometry. Red: Final Astrometry",
+                   KAPA_LABEL_XP);
+    KapaPrepPlot (kapa, cornerPo->n, &graphdata);
+    KapaPlotVector (kapa, cornerPo->n, cornerPo->data.F32, "x");
+    KapaPlotVector (kapa, cornerQo->n, cornerQo->data.F32, "y");
+
+    // New Corners
+    graphdata.color = KapaColorByName("red");
+    graphdata.ptype = 7;
+    graphdata.size = 1.5;
+    KapaPrepPlot (kapa, cornerPn->n, &graphdata);
+    KapaPlotVector (kapa, cornerPn->n, cornerPn->data.F32, "x");
+    KapaPlotVector (kapa, cornerQn->n, cornerQn->data.F32, "y");
+
+    // Residuals
+    psVector *xResid = psVectorAlloc(cornerPn->n, PS_DATA_F32);
+    psVector *yResid = psVectorAlloc(cornerQn->n, PS_DATA_F32);
+    for(int i=0; i < cornerPn->n; i++) {
+        xResid->data.F32[i] = (cornerPd->data.F32[i]);
+        yResid->data.F32[i] = (cornerQd->data.F32[i]);
+    }
+
+    graphdata.color = KapaColorByName("black");
+    graphdata.size=0.5;
+    section.x = 0.3;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a1");
+    KapaSetSection (kapa, &section);
+    psFree(section.name);
+
+    psastroVisualScaleGraphdata (&graphdata, xResid, yResid, true);
+    KapaSetLimits (kapa, &graphdata);
+    KapaBox (kapa, &graphdata);
+    KapaSendLabel (kapa, "dP", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "dQ", KAPA_LABEL_YM);
+    KapaSendLabel (kapa,
+                   "Residual of the Fit from the Initial Astrometry to the Final Astrometry",
+                   KAPA_LABEL_XP);
+    KapaPrepPlot (kapa, cornerPd->n, &graphdata);
+    KapaPlotVector (kapa, cornerPd->n, xResid->data.F32, "x");
+    KapaPlotVector (kapa, cornerQd->n, yResid->data.F32, "y");
+
+    psFree(xResid);
+    psFree(yResid);
+    psastroVisualAskUser (&plotAstromGuessCheck);
+    return true;
+}
+
+
+/**
+ * Plots the pixel scales of the fpa before they are
+ * equalized in psastroMosaicCommonScale
+ */
+bool psastroVisualPlotCommonScale (pmFPA *fpa,         ///< the fpa
+                                   psVector *oldScale  ///< the old pixel scale of each chip in the fpa
+                                   )
+{
+    //make sure we want to plot this
+    if (!isVisual || !plotCommonScale) return false;
+
+    if (!psastroVisualInitWindow(&kapa, "psastro:plots")) return false;
+
+    KapaSection section = {"s1", .05, .05, .9, .9};
+    Graphdata graphdata;
+    psPlane ptCH, ptFP;
+    ptCH.x = 0;
+    ptCH.y = 0;
+    psVector *xVec = psVectorAlloc (oldScale->n, PS_TYPE_F32);
+    psVector *yVec = psVectorAlloc (oldScale->n, PS_TYPE_F32);
+
+    int nobj = 0;
+
+    //project each chip corner to the Focal Plane
+    for(int i = 0; i < fpa->chips->n; i++) {
+        pmChip *chip = fpa->chips->data[i];
+        if (!chip->process || !chip->file_exists) { continue; }
+        if (!chip->toFPA) { continue; }
+
+        psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH);
+        xVec->data.F32[nobj] = ptFP.x;
+        yVec->data.F32[nobj] = ptFP.y;
+        nobj++;
+    }
+
+    //set up plot window
+    KapaInitGraph (&graphdata);
+    KapaClearPlots (kapa);
+    KapaSetSection (kapa, &section);
+    KapaSetFont (kapa, "helvetica", 14);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, oldScale, false);
+    KapaSendLabel (kapa, "L (FP)", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "M (FP)", KAPA_LABEL_YM);
+    KapaSendLabel (kapa, "Old Pixel Scale of FPA Chips (Not to Scale)", KAPA_LABEL_XP);
+
+    psastroVisualAskUser (&plotCommonScale);
+    return true;
+}
+
+
+/**
+ *   plot the residuals between raw stars and ref stars after
+ *   fitting in psastroMosaicOneChip
+ */
+bool psastroVisualPlotMosaicOneChip (psArray *rawstars, psArray *refstars,
+                                     psArray *match, psMetadata *recipe)
+{
+
+    //make sure we want to plot this
+    if (!isVisual || !plotMosaicOneChip) return false;
+
+    //plot the residuals
+    if (!psastroVisualResidPlot(rawstars, refstars, match, recipe, "Single Chip Fit Residuals - Mosaic Mode"))
+        return false;
+
+    //ask for user input and finish
+    psastroVisualAskUser(&plotMosaicOneChip);
+
+    return true;
+}
+
+/** psastroVisualPlotMosaicMatches
+ * Plot the matches between raw and reference stars during psastroVisualMosaicSetMatch
+ */
+bool psastroVisualPlotMosaicMatches( psArray *rawstars, psArray *refstars,
+                                    psArray *match, int iteration,
+                                    psMetadata *recipe)
+{
+    //make sure we want to plot this
+    if (!isVisual || !plotMosaicMatches) return true;
+
+    char title[60];
+    sprintf(title, "Matches found during psastroMosaicSetMatch iteration %d", iteration);
+
+    if (!psastroVisualResidPlot(rawstars, refstars, match, recipe, title))
+        return false;
+
+    //ask for user input
+    psastroVisualAskUser (&plotMosaicMatches);
+    return true;
+}
+
+
+/*********************/
+/** Helper Routines **/
+/*********************/
+
+
+/** psastroVisualInitGraph (kapa, *section, *graphdata)
+ * Initializes graph, sets the section, sets the font,
+ * sets the limits, draws the box
+ */
+bool psastroVisualInitGraph (int kapa, KapaSection *section, Graphdata *graphdata)
+{
+    KapaSetSection (kapa, section);
+    KapaSetFont (kapa, "helvetica", 14);
+    KapaSetLimits (kapa, graphdata);
+    KapaBox (kapa, graphdata);
+    return true;
+}
+
+
+/** psastroVisualTriplePlot
+ * plot 2 vectors whose point sizes are scaled by a third vector
+ * autoscales the plot
+ */
+bool psastroVisualTriplePlot (int kapid, Graphdata *graphdata, psVector *xVec, psVector *yVec, psVector *zVec, bool increasing)
+{
+    psastroVisualScaleGraphdata (graphdata, xVec, yVec, true);
+    //printf("%f %f %f %f \n",graphdata->xmin, graphdata->xmax, graphdata->ymin, graphdata->ymax);
+    // set the scale vector
+    psVector *zScale = psVectorAlloc (zVec->n, PS_DATA_F32);
+    psastroVisualCreateScaleVec (zVec, zScale, increasing);
+
+    KapaSetFont (kapid, "helvetica", 14);
+    KapaSetLimits(kapid, graphdata);
+    KapaBox (kapid, graphdata);
+
+    // the point size will be scaled from the z vector
+    graphdata->ptype = 7;
+    graphdata->style = 2;
+    graphdata->size = -1;
+    KapaPrepPlot (kapid, xVec->n, graphdata);
+    KapaPlotVector (kapid, xVec->n, xVec->data.F32, "x");
+    KapaPlotVector (kapid, yVec->n, yVec->data.F32, "y");
+    KapaPlotVector (kapid, zVec->n, zScale->data.F32, "z");
+    psFree (zScale);
+    return true;
+}
+
+
+/** psastroVisualTripleOverplot
+ * same as above, but does not rescale the plot or redraw the bounding box
+ */
+bool psastroVisualTripleOverplot (int kapid, Graphdata *graphdata, psVector *xVec, psVector *yVec, psVector *zVec, bool increasing) {
+
+    // set the scale vector
+    psVector *zScale = psVectorAlloc (zVec->n, PS_DATA_F32);
+    psastroVisualCreateScaleVec (zVec, zScale, increasing);
+
+    KapaSetFont (kapid, "helvetica", 14);
+
+    // the point size will be scaled from the z vector
+    graphdata->size = -1;
+    KapaPrepPlot (kapid, xVec->n, graphdata);
+    KapaPlotVector (kapid, xVec->n, xVec->data.F32, "x");
+    KapaPlotVector (kapid, yVec->n, yVec->data.F32, "y");
+    KapaPlotVector (kapid, zVec->n, zScale->data.F32, "z");
+    psFree (zScale);
+    return true;
+}
+
+
+/** psastroVisualCreateScaleVec
+ * create a vector of plot point sizes from a vector
+ */
+bool psastroVisualCreateScaleVec (psVector *zVec, psVector *zScale, bool increasing) {
+    // set limits based on data values
+    float zmin = +FLT_MAX;
+    float zmax = -FLT_MAX;
+
+    for (int i = 0; i < zVec->n; i++) {
+        zmin = PS_MIN (zmin, zVec->data.F32[i]);
+        zmax = PS_MAX (zmax, zVec->data.F32[i]);
+    }
+
+    float range = zmax - zmin;
+    if (range == 0.0) {
+        psVectorInit (zScale, 1.0);
+    } else {
+        for (int i = 0; i < zVec->n; i++) {
+            if (increasing) {
+                zScale->data.F32[i] = PS_MIN (1.5, PS_MAX(0.05, 1.5*(zVec->data.F32[i] - zmin)/range));
+            } else {
+                zScale->data.F32[i] = PS_MIN (1.5, PS_MAX(0.05, 1.5*(zmax - zVec->data.F32[i])/range));
+            }
+        }
+    }
+    return true;
+}
+
+
+/** psastroVisualScaleGraphdata
+ * Scale the graphdata structure based on x and y coordinates. Use sigma clipping to
+ * prevent outliers from making te plot region too big.
+ */
+bool psastroVisualScaleGraphdata(Graphdata *graphdata, psVector *xVec, psVector *yVec, bool clip) {
+
+    graphdata->xmin = +FLT_MAX;
+    graphdata->xmax = -FLT_MAX;
+    graphdata->ymin = +FLT_MAX;
+    graphdata->ymax = -FLT_MAX;
+
+    //determine standard deviation of xVec and yVec
+    psStats *statsX = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
+    psStats *statsY = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
+    psVectorStats (statsX, xVec, NULL, NULL, 0);
+    psVectorStats (statsY, yVec, NULL, NULL, 0);
+
+    float xhi = +FLT_MAX, xlo = -FLT_MAX, yhi = +FLT_MAX, ylo = -FLT_MAX;
+    if (clip) {
+        xhi  = statsX->sampleMedian + 3 *statsX->sampleStdev;
+        xlo = statsX->sampleMedian - 3 *statsX->sampleStdev;
+        yhi = statsY->sampleMedian + 3 *statsY->sampleStdev;
+        ylo = statsY->sampleMedian - 3 *statsY->sampleStdev;
+    }
+
+    // abort if there is no good data
+    if (!isfinite(xhi) || !isfinite(xlo) || !isfinite(yhi) || !isfinite(ylo)) {
+        graphdata->xmin = -1;
+        graphdata->ymin  = -1;
+        graphdata->xmax = 1;
+        graphdata->ymax = 1;
+        psFree(statsX);
+        psFree(statsY);
+        return false;
+    }
+
+    for(int i = 0; i < xVec->n; i++) {
+        if (!isfinite(xVec->data.F32[i])) continue;
+        if (xVec->data.F32[i] > xhi || xVec->data.F32[i] < xlo) continue;
+        graphdata->xmin = PS_MIN (graphdata->xmin, xVec->data.F32[i]);
+        graphdata->xmax = PS_MAX (graphdata->xmax, xVec->data.F32[i]);
+    }
+
+    for (int i = 0; i < yVec->n; i++) {
+        if (!isfinite(xVec->data.F32[i])) continue;
+        if (yVec->data.F32[i] > yhi || yVec->data.F32[i] < ylo) continue;
+        graphdata->ymin = PS_MIN (graphdata->ymin, yVec->data.F32[i]);
+        graphdata->ymax = PS_MAX (graphdata->ymax, yVec->data.F32[i]);
+    }
+
+    // add a whitespace border
+    float range = graphdata->xmax - graphdata->xmin;
+    if (range == 0) range = 1;
+    graphdata->xmin -= .05 * range;
+    graphdata->xmax += .05 * range;
+
+    range = graphdata->ymax - graphdata->ymin;
+    if (range == 0) range = 1;
+    graphdata->ymin -= .05 * range;
+    graphdata->ymax += .05 * range;
+
+    psFree (statsX);
+    psFree (statsY);
+    return true;
+}
+
+
+/** psastroVisualResdiPlot
+ * Plots the residuals between matched raw and reference stars. Used to assess the quality of an
+ * astrometry solution
+ */
+bool psastroVisualResidPlot (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe, char *title) {
+
+    //set up the first window
+    if (!psastroVisualInitWindow( &kapa, "psastro:plots")) return false;
+
+    //initialize graph information
+    Graphdata graphdata;
+    KapaSection section;
+
+    KapaInitGraph (&graphdata);
+    KapaClearPlots (kapa);
+
+    graphdata.color = KapaColorByName ("black");
+    graphdata.ptype = 7;
+    graphdata.size = 0.5;
+    graphdata.style = 2;
+
+    section.dx = 0.4;
+    section.dy = 0.4;
+
+    //initialize and populate the plotting vectors
+    bool status = false;
+    float iMagMin = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.INST.MAG.MIN");
+    float iMagMax = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.INST.MAG.MAX");
+    float rMagMin = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.REF.MAG.MIN");
+    float rMagMax = psMetadataLookupF32 (&status, recipe, "PSASTRO.PLOT.REF.MAG.MAX");
+
+    psVector *xVec = psVectorAlloc (match->n, PS_TYPE_F32);
+    psVector *yVec = psVectorAlloc (match->n, PS_TYPE_F32);
+    psVector *zVec = psVectorAlloc (match->n, PS_TYPE_F32);
+
+    // X vs dX
+    section.x = 0.0;
+    section.y = 0.5;
+    section.name = NULL;
+    psStringAppend (&section.name, "a0");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    int n = 0;
+    for (int i = 0; i < match->n; i++) {
+        pmAstromMatch *pair = match->data[i];
+        pmAstromObj *raw = rawstars->data[pair->raw];
+        pmAstromObj *ref = refstars->data[pair->ref];
+
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->x;
+        yVec->data.F32[n] = raw->chip->x - ref->chip->x;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "X", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "dX", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // X vs dY
+    section.x = 0.5;
+    section.y = 0.5;
+    section.name = NULL;
+    psStringAppend (&section.name, "a1");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < match->n; i++) {
+        pmAstromMatch *pair = match->data[i];
+        pmAstromObj *raw = rawstars->data[pair->raw];
+        pmAstromObj *ref = refstars->data[pair->ref];
+
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->x;
+        yVec->data.F32[n] = raw->chip->y - ref->chip->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "X", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "dY", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // Y vs dX
+    section.x = 0.0;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a2");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < match->n; i++) {
+        pmAstromMatch *pair = match->data[i];
+        pmAstromObj *raw = rawstars->data[pair->raw];
+        pmAstromObj *ref = refstars->data[pair->ref];
+
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->y;
+        yVec->data.F32[n] = raw->chip->x - ref->chip->x;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "Y", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "dX", KAPA_LABEL_YM);
+    psastroVisualTriplePlot (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    // Y vs dY
+    section.x = 0.5;
+    section.y = 0.0;
+    section.name = NULL;
+    psStringAppend (&section.name, "a3");
+    KapaSetSection (kapa, &section);
+    psFree (section.name);
+
+    n = 0;
+    for (int i = 0; i < match->n; i++) {
+        pmAstromMatch *pair = match->data[i];
+        pmAstromObj *raw = rawstars->data[pair->raw];
+        pmAstromObj *ref = refstars->data[pair->ref];
+
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->y;
+        yVec->data.F32[n] = raw->chip->y - ref->chip->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa, "Y", KAPA_LABEL_XM);
+    KapaSendLabel (kapa, "dY", KAPA_LABEL_YM);
+    pmKapaPlotVectorTriple_AutoLimits_OpenGraph (kapa, &graphdata, xVec, yVec, zVec, false);
+
+    section.x = 0.0;
+    section.y = 0.0;
+    section.dx = 0.95;
+    section.dy = 0.95;
+    section.name = NULL;
+    psStringAppend (&section.name, "a5");
+    KapaSetSection (kapa, &section);
+    KapaSendLabel (kapa, title, KAPA_LABEL_XP);
+    psFree (section.name);
+
+
+    // X vs Y plot (different window)
+    if (!psastroVisualInitWindow( &kapa2, "psastro:plots"))
+        return false;
+
+    KapaInitGraph (&graphdata);
+    KapaClearPlots (kapa2);
+
+    graphdata.color = KapaColorByName ("black");
+    graphdata.ptype = 2;
+    graphdata.style = 2;
+
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+
+    xVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+    yVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+    zVec = psVectorAlloc (rawstars->n, PS_TYPE_F32);
+
+    // X vs Y by mag (raw)
+    n = 0;
+    for (int i = 0; i < rawstars->n; i++) {
+        pmAstromObj *raw = rawstars->data[i];
+        if (!isfinite(raw->Mag)) continue;
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->x;
+        yVec->data.F32[n] = raw->chip->y;
+        zVec->data.F32[n] = raw->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+
+    KapaSendLabel (kapa2, "X", KAPA_LABEL_XM);
+    KapaSendLabel (kapa2, "Y", KAPA_LABEL_YM);
+    KapaSendLabel (kapa2,
+                   "Chip Coordinates. Black = Raw Stars. Red = Ref Stars. Blue = Matched Stars"
+                   , KAPA_LABEL_XP);
+    psastroVisualTriplePlot (kapa2, &graphdata, xVec, yVec, zVec, false);
+
+    // X vs Y by mag (ref)
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+
+    xVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+    yVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+    zVec = psVectorAlloc (refstars->n, PS_TYPE_F32);
+
+    graphdata.color = KapaColorByName ("red");
+    graphdata.ptype = 7;
+    graphdata.style = 2;
+
+    n = 0;
+    for (int i = 0; i < refstars->n; i++) {
+        pmAstromObj *ref = refstars->data[i];
+        if (!isfinite(ref->Mag)) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = ref->chip->x;
+        yVec->data.F32[n] = ref->chip->y;
+        zVec->data.F32[n] = ref->Mag;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+    psastroVisualTripleOverplot (kapa2, &graphdata, xVec, yVec, zVec, false);
+
+    //rescale the graph to include all points
+    float xmin = graphdata.xmin;
+    float ymin = graphdata.ymin;
+    float xmax = graphdata.xmax;
+    float ymax = graphdata.ymax;
+    psastroVisualScaleGraphdata(&graphdata, xVec, yVec, true);
+    graphdata.xmin = PS_MIN(xmin, graphdata.xmin);
+    graphdata.ymin = PS_MIN(ymin, graphdata.ymin);
+    graphdata.xmax = PS_MAX(xmax, graphdata.xmax);
+    graphdata.ymax = PS_MAX(ymax, graphdata.ymax);
+    KapaSetLimits (kapa2, &graphdata);
+
+    //overplot matched stars in blue
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+
+    xVec = psVectorAlloc (match->n, PS_TYPE_F32);
+    yVec = psVectorAlloc (match->n, PS_TYPE_F32);
+    zVec = psVectorAlloc (match->n, PS_TYPE_F32);
+
+    graphdata.color = KapaColorByName ("blue");
+    n = 0;
+    for (int i = 0; i < match->n; i++) {
+        pmAstromMatch *pair = match->data[i];
+        pmAstromObj *raw = rawstars->data[pair->raw];
+        pmAstromObj *ref = refstars->data[pair->ref];
+        if (raw->Mag < iMagMin) continue;
+        if (raw->Mag > iMagMax) continue;
+        if (ref->Mag < rMagMin) continue;
+        if (ref->Mag > rMagMax) continue;
+
+        xVec->data.F32[n] = raw->chip->x;
+        yVec->data.F32[n] = raw->chip->y;
+        zVec->data.F32[n] = iMagMin;
+        n++;
+    }
+    xVec->n = yVec->n = zVec->n = n;
+    psastroVisualTripleOverplot (kapa2, &graphdata, xVec, yVec, zVec, false);
+
+    psFree (xVec);
+    psFree (yVec);
+    psFree (zVec);
+    return true;
+}
+
+// END OF PROGRAM
+#else
+
+bool psastroSetVisual (bool mode) {return true};
+bool psastroVisualClose() {return true};
+bool psastroVisualPlotLuminosityFunction (psVector *lnMag, psVector *Mag, pmLumFunc *lumFunc, pmLumFunc *rawFunc) {return true};
+bool psastroVisualPlotRawStars (psArray *rawstars, pmFPA *fpa, pmChip *chip, psMetadata *recipe) {return true};
+bool psastroVisualPlotRefStars (psArray *refstars, psMetadata *recipe) {return true};
+bool psastroVisualPlotRemoveClumps (psArray *input, psImage *count, int scale, float limit) {return true};
+bool psastroVisualPlotFixChips (pmFPAfile *input, psVector *xOld, psVector *yOld) {return true};
+bool psastroVisualPlotOneChipFit (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe) {return true};
+bool psastroVisualPlotAstromGuessCheck (psVector *cornerPo, psVector *cornerQo, psVector *cornerPn, psVector *cornerQn, psVector *cornerPd, psVector *cornerQd) {return true};
+bool psastroVisualPlotMosaicOneChip (psArray *rawstars, psArray *refstars, psArray *match, psMetadata *recipe) {return true};
+bool psastroVisualPlotCommonScale (pmFPA *fpa, psVector *oldScale) {return true};
+bool psastroVisualPlotMosaicMatches (psArray *rawstars, psArray *refstars, psArray *match, int iteration, psMetadata *recipe) {return true};
+
+# endif
+
Index: branches/bills_081204/psphot/src/psphotAddNoise.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotAddNoise.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotAddNoise.c	(revision 20890)
@@ -41,6 +41,7 @@
     float GAIN = psMetadataLookupF32(&status, readout->parent->concepts, "CELL.GAIN"); // Cell gain
     PS_ASSERT (status, false);
-
-    FACTOR /= GAIN;
+    if (isfinite(GAIN)) {
+	FACTOR /= GAIN;
+    }
 
     // loop over all source
Index: branches/bills_081204/psphot/src/psphotApResid.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotApResid.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotApResid.c	(revision 20890)
@@ -22,12 +22,12 @@
     bool measureAptrend = psMetadataLookupBool (&status, recipe, "MEASURE.APTREND");
     if (!measureAptrend) {
-	// save nan values since these were not calculated
-	psMetadataAdd (recipe, PS_LIST_TAIL, "SKYBIAS",  PS_DATA_F32 | PS_META_REPLACE, "aperture sky bias",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "SKYSAT",   PS_DATA_F32 | PS_META_REPLACE, "aperture-determined saturation",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "APMIFIT",  PS_DATA_F32 | PS_META_REPLACE, "aperture residual",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "DAPMIFIT", PS_DATA_F32 | PS_META_REPLACE, "ap residual scatter", NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "APLOSS",   PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "number of apresid stars", 0);
-	return true;
+        // save nan values since these were not calculated
+        psMetadataAdd (recipe, PS_LIST_TAIL, "SKYBIAS",  PS_DATA_F32 | PS_META_REPLACE, "aperture sky bias",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "SKYSAT",   PS_DATA_F32 | PS_META_REPLACE, "aperture-determined saturation",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "APMIFIT",  PS_DATA_F32 | PS_META_REPLACE, "aperture residual",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "DAPMIFIT", PS_DATA_F32 | PS_META_REPLACE, "ap residual scatter", NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "APLOSS",   PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "number of apresid stars", 0);
+        return true;
     }
 
@@ -93,5 +93,5 @@
         if (!pmSourceMagnitudes (source, psf, photMode, maskVal)) {
             Nskip ++;
-	    psTrace ("psphot", 3, "skip : bad source mag");
+            psTrace ("psphot", 3, "skip : bad source mag");
             continue;
         }
@@ -99,19 +99,19 @@
         if (!isfinite(source->apMag) || !isfinite(source->psfMag)) {
             Nfail ++;
-	    psTrace ("psphot", 3, "fail : nan mags : %f %f", source->apMag, source->psfMag);
+            psTrace ("psphot", 3, "fail : nan mags : %f %f", source->apMag, source->psfMag);
             continue;
         }
 
-	// aperture residual for this source
-	float dap = source->apMag - source->psfMag;
+        // aperture residual for this source
+        float dap = source->apMag - source->psfMag;
 
         // sanity clipping : if the model is very discrepant, put your expectation in the recipe
         if ((MAX_AP_OFFSET > 0) && (fabs(dap) > MAX_AP_OFFSET)) {
             Nfail ++;
-	    psTrace ("psphot", 3, "fail : bad dap %f %f", dap, MAX_AP_OFFSET);
+            psTrace ("psphot", 3, "fail : bad dap %f %f", dap, MAX_AP_OFFSET);
             continue;
         }
 
-	mag->data.F32[Npsf]     = source->psfMag;
+        mag->data.F32[Npsf]     = source->psfMag;
         apResid->data.F32[Npsf] = dap;
         xPos->data.F32[Npsf]    = model->params->data.F32[PM_PAR_XPOS];
@@ -130,5 +130,5 @@
         Npsf ++;
     }
-    
+
     psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "measure aperture residuals for %d objects (%d skipped, %d failed, %ld invalid)\n",
               Npsf, Nskip, Nfail, sources->n - Npsf - Nskip - Nfail);
@@ -137,18 +137,18 @@
     if (Npsf < APTREND_NSTAR_MIN) {
         psWarning("Only %d valid aperture residual sources (need %d), giving up", Npsf, APTREND_NSTAR_MIN);
-	// save nan values since these were not calculated
-	psMetadataAdd (recipe, PS_LIST_TAIL, "SKYBIAS",  PS_DATA_F32 | PS_META_REPLACE, "aperture sky bias",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "SKYSAT",   PS_DATA_F32 | PS_META_REPLACE, "aperture-determined saturation",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "APMIFIT",  PS_DATA_F32 | PS_META_REPLACE, "aperture residual",   NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "DAPMIFIT", PS_DATA_F32 | PS_META_REPLACE, "ap residual scatter", NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "APLOSS",   PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", NAN);
-	psMetadataAdd (recipe, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "number of apresid stars", 0);
-
-	psFree (mag);
-	psFree (mask);
-	psFree (xPos);
-	psFree (yPos);
-	psFree (apResid);
-	psFree (dMag);
+        // save nan values since these were not calculated
+        psMetadataAdd (recipe, PS_LIST_TAIL, "SKYBIAS",  PS_DATA_F32 | PS_META_REPLACE, "aperture sky bias",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "SKYSAT",   PS_DATA_F32 | PS_META_REPLACE, "aperture-determined saturation",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "APMIFIT",  PS_DATA_F32 | PS_META_REPLACE, "aperture residual",   NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "DAPMIFIT", PS_DATA_F32 | PS_META_REPLACE, "ap residual scatter", NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "APLOSS",   PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", NAN);
+        psMetadataAdd (recipe, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "number of apresid stars", 0);
+
+        psFree (mag);
+        psFree (mask);
+        psFree (xPos);
+        psFree (yPos);
+        psFree (apResid);
+        psFree (dMag);
         return false;
     }
@@ -168,19 +168,19 @@
     for (int i = 1; i <= APTREND_ORDER_MAX; i++) {
 
-	if (!psphotApResidTrend (readout, psf, Npsf, i, &errorScale, &errorFloor, mask, xPos, yPos, apResid, dMag)) {
-	    break;
-	}
-
-	// store the resulting errorFloor values and the scales, redo the best
-	if (errorFloor < errorFloorMin) {
-	    errorFloorMin = errorFloor;
-	    entryMin = i;
-	}
+        if (!psphotApResidTrend (readout, psf, Npsf, i, &errorScale, &errorFloor, mask, xPos, yPos, apResid, dMag)) {
+            break;
+        }
+
+        // store the resulting errorFloor values and the scales, redo the best
+        if (errorFloor < errorFloorMin) {
+            errorFloorMin = errorFloor;
+            entryMin = i;
+        }
     }
     if (entryMin == -1) {
-	psAbort ("failed on ApResid Trend");
-    }
-
-    // XXX catch error condition 
+        psAbort ("failed on ApResid Trend");
+    }
+
+    // XXX catch error condition
     psphotApResidTrend (readout, psf, Npsf, entryMin, &errorScale, &errorFloor, mask, xPos, yPos, apResid, dMag);
 
@@ -190,14 +190,14 @@
     psVector *dMagSys = (psVector *) psBinaryOp (NULL, (void *) dMag, "*", (void *) psScalarAlloc(errorScale, PS_TYPE_F32));
 
-    if (psTraceGetLevel("psphot") >= 5) {
-	FILE *dumpFile = fopen ("apresid.dat", "w");
-	for (int i = 0; i < xPos->n; i++) {
-	    fprintf (dumpFile, "%f %f  %f %f %f  %f %f  %x\n", 
-		     xPos->data.F32[i], yPos->data.F32[i], 
-		     mag->data.F32[i], dMag->data.F32[i], dMagSys->data.F32[i],
-		     apResid->data.F32[i], apResidRes->data.F32[i],
-		     mask->data.U8[i]);
-	}
-	fclose (dumpFile);
+    if (psTraceGetLevel("psphot") >= 2) {
+        FILE *dumpFile = fopen ("apresid.dat", "w");
+        for (int i = 0; i < xPos->n; i++) {
+            fprintf (dumpFile, "%f %f  %f %f %f  %f %f  %x\n",
+                     xPos->data.F32[i], yPos->data.F32[i],
+                     mag->data.F32[i], dMag->data.F32[i], dMagSys->data.F32[i],
+                     apResid->data.F32[i], apResidRes->data.F32[i],
+                     mask->data.U8[i]);
+        }
+        fclose (dumpFile);
     }
 
@@ -252,5 +252,5 @@
     int nBin = PS_MAX (dMag->n / nGroup, 1);
 
-    // output vectors for ApResid trend 
+    // output vectors for ApResid trend
     psVector *dSo = psVectorAlloc (nBin, PS_TYPE_F32);
     psVector *dMo = psVectorAlloc (nBin, PS_TYPE_F32);
@@ -267,46 +267,47 @@
     int n = 0;
     for (int i = 0; i < dMo->n; i++) {
-	int j;
-	for (j = 0; (j < nGroup) && (n < dMag->n); j++, n++) {
-	    int N = index->data.U32[n];
-	    dMSubset->data.F32[j] = dMag->data.F32[N];
-	    dASubset->data.F32[j] = dap->data.F32[N];
-	    mkSubset->data.U8[j]  = mask->data.U8[N];
-	}
-	dMSubset->n = j;
-	dASubset->n = j;
-	mkSubset->n = j;
-
-	psStatsInit (statsS);
-	psStatsInit (statsM);
-
-	if (j > 2) {
-	    bool status = true;
-	    status &= psVectorStats (statsS, dASubset, NULL, mkSubset, 0xff);
-	    status &= psVectorStats (statsM, dMSubset, NULL, mkSubset, 0xff);
-	    if (!status) { psErrorClear (); }
-	    dSo->data.F32[i] = statsS->robustStdev;
-	    dMo->data.F32[i] = statsM->sampleMean;
-	    dRo->data.F32[i] = statsS->robustStdev / statsM->sampleMean;
-	} else {
-	    dSo->data.F32[i] = NAN;
-	    dMo->data.F32[i] = NAN;
-	    dRo->data.F32[i] = NAN;
-	}
+        int j;
+        for (j = 0; (j < nGroup) && (n < dMag->n); j++, n++) {
+            int N = index->data.U32[n];
+            dMSubset->data.F32[j] = dMag->data.F32[N];
+            dASubset->data.F32[j] = dap->data.F32[N];
+            mkSubset->data.U8[j]  = mask->data.U8[N];
+        }
+        dMSubset->n = j;
+        dASubset->n = j;
+        mkSubset->n = j;
+
+        psStatsInit (statsS);
+        psStatsInit (statsM);
+
+        if (j > 2) {
+            bool status = true;
+            status &= psVectorStats (statsS, dASubset, NULL, mkSubset, 0xff);
+            status &= psVectorStats (statsM, dMSubset, NULL, mkSubset, 0xff);
+            if (!status) { psErrorClear (); }
+            dSo->data.F32[i] = statsS->robustStdev;
+            dMo->data.F32[i] = statsM->sampleMean;
+            dRo->data.F32[i] = statsS->robustStdev / statsM->sampleMean;
+            //      fprintf (stderr, "%d (%d) : sys: %f, phot: %f, rat: %f\n", i, j, dSo->data.F32[i], dMo->data.F32[i], dRo->data.F32[i]);
+        } else {
+            dSo->data.F32[i] = NAN;
+            dMo->data.F32[i] = NAN;
+            dRo->data.F32[i] = NAN;
+        }
     }
     psFree (dMSubset);
     psFree (dASubset);
     psFree (mkSubset);
-    
+
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN);
     if (!psVectorStats (stats, dRo, NULL, NULL, 0)) {
-	// XXX better testing of raised error
-	psErrorClear();
+        // XXX better testing of raised error
+        psErrorClear();
     }
 
     *errorScale = stats->sampleMedian;
     for (int i = 0; i < dSo->n; i++) {
-	*errorFloor = dSo->data.F32[i];
-	if (isfinite(*errorFloor)) break;
+        *errorFloor = dSo->data.F32[i];
+        if (isfinite(*errorFloor)) break;
     }
 
@@ -317,5 +318,5 @@
     psFree (dMo);
     psFree (dSo);
-    
+
     psFree (statsS);
     psFree (statsM);
@@ -327,19 +328,20 @@
 
     int Nx, Ny;
-	
+
     if (readout->image->numCols > readout->image->numRows) {
         Nx = scale;
-	float AR = readout->image->numRows / (float) readout->image->numCols;
-	Ny = (int) (Nx * AR + 0.5);
+        float AR = readout->image->numRows / (float) readout->image->numCols;
+        Ny = (int) (Nx * AR + 0.5);
         Ny = PS_MAX (1, Ny);
     } else {
-	Ny = scale;
-	float AR = readout->image->numRows / (float) readout->image->numCols;
-	Nx = (int) (Ny * AR + 0.5);
-	Nx = PS_MAX (1, Nx);
-    }
-
-    if (Npsf < 3*Nx*Ny) {
-	return false;
+        Ny = scale;
+        float AR = readout->image->numRows / (float) readout->image->numCols;
+        Nx = (int) (Ny * AR + 0.5);
+        Nx = PS_MAX (1, Nx);
+    }
+
+    // require at least 10 stars per spatial bin
+    if (Npsf < 10*Nx*Ny) {
+        return false;
     }
 
@@ -356,7 +358,13 @@
     psf->ApTrend = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats);
 
+    // XXX somewhat arbitrary: soften the errors so the few bright stars do not totally dominate:
+    psVector *dMagSoft = psVectorAlloc (dMag->n, PS_TYPE_F32);
+    for (int i = 0; i < dMag->n; i++) {
+        dMagSoft->data.F32[i] = hypot(dMag->data.F32[i], 0.01);
+    }
+
     // XXX test for errors here
-    pmTrend2DFit (psf->ApTrend, mask, 0xff, xPos, yPos, apResid, dMag);
-    
+    pmTrend2DFit (psf->ApTrend, mask, 0xff, xPos, yPos, apResid, dMagSoft);
+
     // construct the fitted values and the residuals
     psVector *apResidFit = pmTrend2DEvalVector (psf->ApTrend, xPos, yPos);
@@ -373,4 +381,5 @@
 
     psFree (stats);
+    psFree (dMagSoft);
     psFree (apResidFit);
     psFree (apResidRes);
@@ -378,3 +387,3 @@
     return true;
 }
-    
+
Index: branches/bills_081204/psphot/src/psphotArguments.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotArguments.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotArguments.c	(revision 20890)
@@ -84,4 +84,5 @@
         psArgumentRemove (N, &argc, argv);
 	psphotSetVisual (true);
+	// pmSourceSetVisual (true);
     }
 
Index: branches/bills_081204/psphot/src/psphotFindFootprints.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotFindFootprints.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotFindFootprints.c	(revision 20890)
@@ -12,7 +12,7 @@
     float FOOTPRINT_NSIGMA_LIMIT;
     if (pass == 1) {
-	FOOTPRINT_NSIGMA_LIMIT = psMetadataLookupS32(&status, recipe, "FOOTPRINT_NSIGMA_LIMIT");
+        FOOTPRINT_NSIGMA_LIMIT = psMetadataLookupS32(&status, recipe, "FOOTPRINT_NSIGMA_LIMIT");
     } else {
-	FOOTPRINT_NSIGMA_LIMIT = psMetadataLookupS32(&status, recipe, "FOOTPRINT_NSIGMA_LIMIT_2");
+        FOOTPRINT_NSIGMA_LIMIT = psMetadataLookupS32(&status, recipe, "FOOTPRINT_NSIGMA_LIMIT_2");
     }
     PS_ASSERT (status, false);
@@ -25,11 +25,11 @@
     int growRadius = 0;
     if (pass == 1) {
-	growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS");
+        growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS");
     } else {
-	growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS_2");
+        growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS_2");
     }
     PS_ASSERT (status, false);
 
-    // find the raw footprints & assign the peaks to those footprints 
+    // find the raw footprints & assign the peaks to those footprints
     psArray *footprints = pmFootprintsFind (significance, threshold, npixMin);
 
@@ -38,5 +38,5 @@
 
     // footprints now owns the peaks; after culling (below), we will rebuild the peaks array
-    psFree (detections->peaks); 
+    psFree (detections->peaks);
 
     psLogMsg ("psphot", PS_LOG_MINUTIA, "found %ld footprints: %f sec\n", footprints->n, psTimerMark ("psphot.footprints"));
@@ -44,24 +44,26 @@
     // optionally grow footprints isotropically by growRadius pixels
     if (growRadius > 0) {
-	psArray *tmp = pmFootprintArrayGrow(footprints, growRadius);
-	psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp->n, psTimerMark ("psphot.footprints"));
-	psFree(footprints);
-	footprints = tmp;
+        bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolve
+        psArray *tmp = pmFootprintArrayGrow(footprints, growRadius);
+        psImageConvolveSetThreads(oldThreads);
+        psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp->n, psTimerMark ("psphot.footprints"));
+        psFree(footprints);
+        footprints = tmp;
     }
 
     if (pass == 2) {
-	// merge in old peaks;
-	const int includePeaks = 0x0 | 0x2; // i.e. just from newFootprints
-	
-	// XXX EAM : still not sure I understand this: are we double-couning or undercounting peaks
+        // merge in old peaks;
+        const int includePeaks = 0x0 | 0x2; // i.e. just from newFootprints
 
-	psArray *mergedFootprints = pmFootprintArraysMerge(detections->footprints, footprints, includePeaks);
-	psLogMsg ("psphot", PS_LOG_MINUTIA, "merged %ld new footprints with %ld existing ones: %f sec\n", footprints->n, detections->footprints->n, psTimerMark ("psphot.footprints"));
+        // XXX EAM : still not sure I understand this: are we double-couning or undercounting peaks
 
-	psFree(footprints);
-	psFree(detections->footprints);
-	detections->footprints = mergedFootprints;
+        psArray *mergedFootprints = pmFootprintArraysMerge(detections->footprints, footprints, includePeaks);
+        psLogMsg ("psphot", PS_LOG_MINUTIA, "merged %ld new footprints with %ld existing ones: %f sec\n", footprints->n, detections->footprints->n, psTimerMark ("psphot.footprints"));
+
+        psFree(footprints);
+        psFree(detections->footprints);
+        detections->footprints = mergedFootprints;
     } else {
-	detections->footprints = footprints;
+        detections->footprints = footprints;
     }
 
Index: branches/bills_081204/psphot/src/psphotImageLoop.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotImageLoop.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotImageLoop.c	(revision 20890)
@@ -72,4 +72,14 @@
 		}
 	    }
+
+	    status = true;
+	    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL");
+	    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV");
+	    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND");
+	    if (!status) {
+		psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files");
+		psFree (view);
+		return false;
+	    }
 	}
 
Index: branches/bills_081204/psphot/src/psphotModelBackground.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotModelBackground.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotModelBackground.c	(revision 20890)
@@ -145,5 +145,5 @@
             // convert the ruff grid cell to the equivalent fine grid cell
             // XXX we need to watch out for row0,col0
-            ruffRegion = psRegionSet (ix, ix + 2, iy, iy + 2);
+            ruffRegion = psRegionSet (ix, ix + 1, iy, iy + 1);
             fineRegion = psImageBinningSetFineRegion (binning, ruffRegion);
             fineRegion = psRegionForImage (image, fineRegion);
Index: branches/bills_081204/psphot/src/psphotReadout.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotReadout.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotReadout.c	(revision 20890)
@@ -7,5 +7,4 @@
     psphotModelClassInit ();            // load implementation-specific models
     psphotSetThreads ();
-    psImageConvolveSetThreads(true);
     return true;
 }
Index: branches/bills_081204/psphot/src/psphotReadoutCleanup.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotReadoutCleanup.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotReadoutCleanup.c	(revision 20890)
@@ -7,16 +7,4 @@
 
     // remove internal pmFPAfiles, if created
-    bool status = true;
-    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL");
-    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV");
-    status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND");
-    if (!status) {
-	psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files");
-	psFree (psf);
-	psFree (sources);
-	psFree (detections);
-	return false;
-    }
-
     if (psErrorCodeLast() == PSPHOT_ERR_DATA) {
         psErrorStackPrint(stderr, "Error in the psphot readout analysis");
Index: branches/bills_081204/psphot/src/psphotSignificanceImage.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotSignificanceImage.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotSignificanceImage.c	(revision 20890)
@@ -12,5 +12,5 @@
     // smooth the image and weight map
     psTimerStart ("psphot.smooth");
-    psImageConvolveSetThreads(true);
+    bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading in psImageConvolve
 
     // XXX we can a) choose fft to convolve if needed and b) multithread fftw
@@ -128,9 +128,12 @@
 
     psFree(smooth_wt);
+
+    psImageConvolveSetThreads(oldThreads);
+
     return smooth_im;
 }
 
-# if (0) 
-{ 
+# if (0)
+{
     // threadingdemo
     // smooth the image, applying the mask as we go
Index: branches/bills_081204/psphot/src/psphotSourceSize.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotSourceSize.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotSourceSize.c	(revision 20890)
@@ -34,4 +34,10 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSPHOT.CR.GROW is not positive.");
         return false;
+    }
+
+    float soft = psMetadataLookupF32(&status, recipe, "PSPHOT.CR.NSIGMA.SOFTEN"); // Softening parameter
+    if (!status || !isfinite(soft) || soft < 0.0) {
+        psWarning("PSPHOT.CR.NSIGMA.SOFTEN not set; defaulting to zero.");
+        soft = 0.0;
     }
 
@@ -94,23 +100,26 @@
         // Compare the central pixel with those on either side, for the four possible lines through it.
 
+        // Soften weights (add systematic error)
+        float softening = soft * PS_SQR(source->peak->flux); // Softening for weights
+
         // Across the middle: y = 0
         float cX = 2*resid[yPeak][xPeak]   - resid[yPeak+0][xPeak-1]  - resid[yPeak+0][xPeak+1];
         float dcX = 4*weight[yPeak][xPeak] + weight[yPeak+0][xPeak-1] + weight[yPeak+0][xPeak+1];
-        float nX = cX / sqrtf(dcX);
+        float nX = cX / sqrtf(dcX + softening);
 
         // Up the centre: x = 0
         float cY = 2*resid[yPeak][xPeak]   - resid[yPeak-1][xPeak+0]  - resid[yPeak+1][xPeak+0];
         float dcY = 4*weight[yPeak][xPeak] + weight[yPeak-1][xPeak+0] + weight[yPeak+1][xPeak+0];
-        float nY = cY / sqrtf(dcY);
+        float nY = cY / sqrtf(dcY + softening);
 
         // Diagonal: x = y
         float cL = 2*resid[yPeak][xPeak]   - resid[yPeak-1][xPeak-1]  - resid[yPeak+1][xPeak+1];
         float dcL = 4*weight[yPeak][xPeak] + weight[yPeak-1][xPeak-1] + weight[yPeak+1][xPeak+1];
-        float nL = cL / sqrtf(dcL);
+        float nL = cL / sqrtf(dcL + softening);
 
         // Diagonal: x = - y
         float cR = 2*resid[yPeak][xPeak]   - resid[yPeak+1][xPeak-1]  - resid[yPeak-1][xPeak+1];
         float dcR = 4*weight[yPeak][xPeak] + weight[yPeak+1][xPeak-1] + weight[yPeak-1][xPeak+1];
-        float nR = cR / sqrtf(dcR);
+        float nR = cR / sqrtf(dcR + softening);
 
         // P(chisq > chisq_obs; Ndof) = gamma_Q (Ndof/2, chisq/2)
@@ -218,5 +227,7 @@
     // now that we have masked pixels associated with CRs, we can grow the mask
     if (grow > 0) {
+        bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolveMask
         psImage *newMask = psImageConvolveMask(NULL, readout->mask, crMask, crMask, -grow, grow, -grow, grow);
+        psImageConvolveSetThreads(oldThreads);
         if (!newMask) {
             psError(PS_ERR_UNKNOWN, false, "Unable to grow CR mask");
Index: branches/bills_081204/psphot/src/psphotVisual.c
===================================================================
--- branches/cnb_branch_20081104/psphot/src/psphotVisual.c	(revision 20530)
+++ branches/bills_081204/psphot/src/psphotVisual.c	(revision 20890)
@@ -691,11 +691,11 @@
     int DY = 64;
 
-    psImage *psfMosaic = psImageAlloc (3*DX, 3*DY, PS_TYPE_F32);
+    psImage *psfMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32);
     psImageInit (psfMosaic, 0.0);
     
-    psImage *funMosaic = psImageAlloc (3*DX, 3*DY, PS_TYPE_F32);
+    psImage *funMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32);
     psImageInit (funMosaic, 0.0);
 
-    psImage *resMosaic = psImageAlloc (3*DX, 3*DY, PS_TYPE_F32);
+    psImage *resMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32);
     psImageInit (resMosaic, 0.0);
 
@@ -703,9 +703,9 @@
 
     // generate a fake model at each of the 3x3 image grid positions
-    for (int x = -1; x <= +1; x ++) {
-	for (int y = -1; y <= +1; y ++) {
+    for (int x = -2; x <= +2; x ++) {
+	for (int y = -2; y <= +2; y ++) {
 	    // use the center of the center pixel of the image
-	    float xc = (int)((0.5 + 0.45*x)*readout->image->numCols) + readout->image->col0 + 0.5;
-	    float yc = (int)((0.5 + 0.45*y)*readout->image->numRows) + readout->image->row0 + 0.5;
+	    float xc = (int)((0.5 + 0.225*x)*readout->image->numCols) + readout->image->col0 + 0.5;
+	    float yc = (int)((0.5 + 0.225*y)*readout->image->numRows) + readout->image->row0 + 0.5;
 
 	    // assign the x and y coords to the image center
@@ -723,8 +723,7 @@
 	    // no need to mask the source here
 	    // XXX should we measure this for the analytical model only or the full model?
-	    pmModelAddWithOffset (psfMosaic, NULL, model, PM_MODEL_OP_FULL | PM_MODEL_OP_CENTER, 0, -x*DX, -y*DY);
-	    pmModelAddWithOffset (funMosaic, NULL, model, PM_MODEL_OP_FUNC | PM_MODEL_OP_CENTER, 0, -x*DX, -y*DY);
-	    pmModelAddWithOffset (resMosaic, NULL, model, PM_MODEL_OP_RES0 | PM_MODEL_OP_RES1 | PM_MODEL_OP_CENTER, 0, -x*DX, -y*DY);
-
+	    pmModelAddWithOffset (psfMosaic, NULL, model, PM_MODEL_OP_FULL | PM_MODEL_OP_CENTER, 0, x*DX, y*DY);
+	    pmModelAddWithOffset (funMosaic, NULL, model, PM_MODEL_OP_FUNC | PM_MODEL_OP_CENTER, 0, x*DX, y*DY);
+	    pmModelAddWithOffset (resMosaic, NULL, model, PM_MODEL_OP_RES0 | PM_MODEL_OP_RES1 | PM_MODEL_OP_CENTER, 0, x*DX, y*DY);
 	    psFree (model);
 	}
Index: branches/bills_081204/pstamp/src/ppstampMakeStamp.c
===================================================================
--- branches/cnb_branch_20081104/pstamp/src/ppstampMakeStamp.c	(revision 20530)
+++ branches/bills_081204/pstamp/src/ppstampMakeStamp.c	(revision 20890)
@@ -30,14 +30,4 @@
 
     outChip->fromFPA = psPlaneTransformInvert(NULL, outChip->toFPA, *roi, 50);
-
-    // remove these keys which may have been copied from the input header
-    // XXX pmAstromWriteWCS should do this since it's the one that's inserting
-    // the PC00* style keywords
-    if (psMetadataLookup(outHDU->header, "CD1_1")) {
-        psMetadataRemoveKey(outHDU->header, "CD1_1");
-        psMetadataRemoveKey(outHDU->header, "CD1_2");
-        psMetadataRemoveKey(outHDU->header, "CD2_1");
-        psMetadataRemoveKey(outHDU->header, "CD2_2");
-    }
 
     if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_NONLIN_TOL)) {
Index: branches/bills_081204/pswarp/src/pswarpDefineSkycell.c
===================================================================
--- branches/cnb_branch_20081104/pswarp/src/pswarpDefineSkycell.c	(revision 20530)
+++ branches/bills_081204/pswarp/src/pswarpDefineSkycell.c	(revision 20890)
@@ -72,5 +72,5 @@
     skyConfig->arguments = psMemIncrRefCounter(config->arguments);
 
-    format = pmConfigCameraFormatFromHeader (NULL, NULL, skyConfig, phu, false);
+    format = pmConfigCameraFormatFromHeader (NULL, NULL, NULL, skyConfig, phu, false);
     if (!format) {
         psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
@@ -95,5 +95,5 @@
     // load the given filerule (from config->camera) and bind it to the fpa
     // the returned file is just a view to the entry on config->files
-    file = pmFPAfileDefineInput (skyConfig, fpa, filename);
+    file = pmFPAfileDefineInput (skyConfig, fpa, NULL, filename);
     if (!file) {
         psError(PS_ERR_IO, false, "file %s not defined\n", filename);
Index: branches/bills_081204/pswarp/src/pswarpTransformSources.c
===================================================================
--- branches/cnb_branch_20081104/pswarp/src/pswarpTransformSources.c	(revision 20530)
+++ branches/bills_081204/pswarp/src/pswarpTransformSources.c	(revision 20890)
@@ -90,4 +90,8 @@
 
         new->modelPSF = pmModelAlloc(source->modelPSF->type);
+	new->modelPSF->params->data.F32[PM_PAR_I0]=
+	  (isfinite(new->psfMag) ? pow(10.0, -0.4*new->psfMag) : NAN);
+	new->modelPSF->dparams->data.F32[PM_PAR_I0]=
+	  (isfinite(new->psfMag) ? new->errMag*pow(10.0, -0.4*new->psfMag) : NAN);
 
 #if 0
Index: branches/bills_081204/tools/chip_outputs.pl
===================================================================
--- branches/bills_081204/tools/chip_outputs.pl	(revision 20890)
+++ branches/bills_081204/tools/chip_outputs.pl	(revision 20890)
@@ -0,0 +1,88 @@
+#!/usr/bin/env perl
+
+#
+# Copy chip outputs to the current directory
+#
+
+use warnings;
+use strict;
+
+use DBI;
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+use constant EXTENSIONS => { 'IMAGE' => '.ch.fits', # Image
+                             'MASK' => '.ch.mk.fits', # Mask
+                             'WEIGHT' => '.ch.wt.fits', # Weight
+                             'SOURCES' => '.cmf', # Sources
+                         };
+
+my ($db_host, $db_name, $db_user, $db_pw); # Database details
+my ($chip_id, $class_id);       # Chip of interest
+my ($input);                    # Input list
+my ($products);                 # Products of interest
+
+GetOptions(
+           'dbhost=s' => \$db_host, # Database host name
+           'dbname=s' => \$db_name, # Database name
+           'dbuser=s' => \$db_user, # Database user
+           'dbpass=s' => \$db_pw, # Database p/w
+           'chip_id=s' => \$chip_id, # Chip identifier
+           'class_id=s' => \$class_id, # Class identifier
+           'products=s' => \$products, # Products of interest
+           ) or die "Unable to parse arguments.\n";
+die "Unknown option: @ARGV\n" if @ARGV;
+die "Required options: --dbhost --dbname --dbuser --dbpass --chip_id\n"
+    unless defined $db_host
+    and defined $db_name
+    and defined $db_user
+    and defined $db_pw
+    and defined $chip_id;
+
+# Database connection
+my $db = DBI->connect( "DBI:mysql:database=$db_name;host=$db_host;mysql_socket=" . DB_SOCKET(),
+                       $db_user,
+                       $db_pw,
+                       { RaiseError => 1, AutoCommit => 1 }
+                       ) or die "Unable to connect to database: $DBI::errstr";
+
+# Query to run
+my $sql = "SELECT path_base, class_id" .
+    " FROM chipProcessedImfile " .
+    " WHERE chip_id = $chip_id";
+$sql .= " AND class_id = \'$class_id\'" if defined $class_id;
+
+# List of chips
+my $results = $db->selectall_arrayref( $sql ) or die "Unable to execute SQL: $DBI::errstr";
+
+my @products;                   # Array of products
+if (defined $products) {
+    @products = split /,/, $products;
+} else {
+    @products = keys %{EXTENSIONS()};
+}
+
+foreach my $row ( @$results ) {
+    my $path = $$row[0];        # Path base
+    my $class_id = $$row[1];    # Class identifier
+
+    my $name = "$path.$class_id"; # Full name
+    foreach my $product ( @products ) {
+        copy_extension( $name, ${EXTENSIONS()}{$product} );
+    }
+}
+
+# Pau.
+
+
+sub copy_extension
+{
+    my $path = shift;           # Path to copy
+    my $ext = shift;            # Extension to copy
+
+    my $neb = "$path$ext"; # Nebulous file
+    my $source = `neb-locate --path $neb` or die "Unable to locate $neb\n"; # Actual file
+    chomp $source;
+    my ($target) = $source =~ m|^.*:(.*)|; # Target name
+    !system "cp $source $target" or die "Unable to copy $source\n";
+}
Index: branches/bills_081204/tools/diff_outputs.pl
===================================================================
--- branches/cnb_branch_20081104/tools/diff_outputs.pl	(revision 20530)
+++ branches/bills_081204/tools/diff_outputs.pl	(revision 20890)
@@ -2,5 +2,5 @@
 
 #
-# Copy diff inputs to the current directory
+# Copy diff outputs to the current directory
 #
 
@@ -16,4 +16,6 @@
                              'WEIGHT' => '.wt.fits', # Weight
                              'SOURCES' => '.cmf', # Sources
+                             'JPEG1' => '.b1.jpg', # Binned JPEG
+                             'JPEG2' => '.b2.jpg', # Binned JPEG
                          };
 
@@ -21,4 +23,5 @@
 my ($diff_id);                  # Diff of interest
 my ($input);                    # Input list
+my ($products);                 # Products of interest
 
 GetOptions(
@@ -28,4 +31,5 @@
            'dbpass=s' => \$db_pw, # Database p/w
            'diff_id=s' => \$diff_id, # Diff identifier
+           'products=s' => \$products, # Products of interest
            ) or die "Unable to parse arguments.\n";
 die "Unknown option: @ARGV\n" if @ARGV;
@@ -52,23 +56,30 @@
                                      ) or die "Unable to execute SQL: $DBI::errstr";
 
+my @products;                   # Array of products
+if (defined $products) {
+    @products = split /,/, $products;
+} else {
+    @products = keys %{EXTENSIONS()};
+}
+
 foreach my $diff ( @$diffs ) {
-    copy_extensions( $diff, EXTENSIONS() );
+    foreach my $product ( @products ) {
+        copy_extension( $diff, ${EXTENSIONS()}{$product} );
+    }
 }
+
 
 # Pau.
 
 
-sub copy_extensions
+sub copy_extension
 {
     my $path = shift;           # Path to copy
-    my $extensions = shift;     # Extensions structure
+    my $ext = shift;            # Extension to copy
 
-    foreach my $type ( keys %$extensions ) {
-        my $ext = $$extensions{$type}; # Extension
-        my $neb = "$path$ext"; # Nebulous file
-        my $source = `neb-locate --path $neb` or die "Unable to locate $neb\n"; # Actual file
-        chomp $source;
-        my ($target) = $source =~ m|^.*:(.*)|; # Target name
-        !system "cp $source $target" or die "Unable to copy $source\n";
-    }
+    my $file = "$path$ext";     # File
+    my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file
+    chomp $source;
+    my ($target) = $source =~ m|^.*[:/](.*)|; # Target name
+    !system "cp $source $target" or die "Unable to copy $source\n";
 }
Index: branches/bills_081204/tools/ds9_regions_mask.pl
===================================================================
--- branches/bills_081204/tools/ds9_regions_mask.pl	(revision 20890)
+++ branches/bills_081204/tools/ds9_regions_mask.pl	(revision 20890)
@@ -0,0 +1,53 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use constant COLOURS => { 'green' => 0,
+                          'blue' => 1,
+                          'yellow' => 2,
+                          'magenta' => 3,
+                          'cyan' => 4,
+                          'red' => 5,
+                      };
+
+my $regions = shift @ARGV;      # Input regions file
+my $masks = shift @ARGV;        # Output mask description
+
+my $global_colour;              # Default colour
+
+my ($input, $output);           # Files
+open $input, $regions;
+open $output, "> $masks";
+while (<$input>) {
+    s/^\#.*//;                   # Comments
+    next unless /\S+/;          # Blank line
+    if (/^global.*color=(\S+)/) {
+        $global_colour = $1;
+        next;
+    }
+    next if /^physical/;        # Boring line
+    if (/^circle\((\S+),(\S+),(\S+)\)/) {
+        my $x = $1;             # x coord
+        my $y = $2;             # y coord
+        my $rad = $3;           # radius
+        my $col;                # Colour
+        if (/color=(\S+)/) {
+            $col = $1;
+        } else {
+            $col = $global_colour;
+        }
+        die "Unknown colour." unless defined $col;
+
+        my $code = ${COLOURS()}{$col};
+        die "Unrecognised colour: $col" unless defined $code;
+
+        print $output "$code $x $y $rad\n";
+        next;
+    }
+    die "Unrecognised line: $_";
+}
+close $input;
+close $output;
+
+__END__
Index: branches/bills_081204/tools/manual_masking.mana
===================================================================
--- branches/bills_081204/tools/manual_masking.mana	(revision 20890)
+++ branches/bills_081204/tools/manual_masking.mana	(revision 20890)
@@ -0,0 +1,654 @@
+$KAPA = unix://kapa
+
+macro fixmask.new
+  if (($0 != 2) && ($0 != 3))
+    echo "USAGE: fixmask.new (chip) [cell]"
+    echo "subtract 2 from last line number for next cell"
+    break
+  end
+
+  dev -n 0
+  dev -n 1
+  sleep 1
+
+  dev -n 0
+  showchips $1
+  # resize -n 0 1150 1000
+  # center {4846/2} {4868/2} -5
+
+  $outroot = GPC1.MASK.20081109
+  $chip = $1
+
+  $startcell = -1
+  if ($0 == 3)
+    $startcell = $2
+  else
+    file $outroot.$chip.fits exists
+    if ($exists)
+     echo "$outroot.$chip.fits exists, need to specify starting cell"
+     break
+    end
+  end
+
+  for i $startcell 64
+    if ($i == -1)
+      rd mkc GPC1.MASK.CTE.$chip.fits
+      keyword mkc NEXTEND -wd 0
+
+      # add the mask bit names here
+      keyword mkc MSKNUM   -wd 3
+      keyword mkc MSKNAM00 -w DETECTOR
+      keyword mkc MSKVAL00 -wd 2
+      keyword mkc MSKNAM01 -w FLAT
+      keyword mkc MSKVAL01 -wd 4
+      keyword mkc MSKNAM02 -w DARK
+      keyword mkc MSKVAL02 -wd 64
+
+      wd mkc $outroot.$chip.fits
+      continue
+    else
+      rd mkc GPC1.MASK.CTE.$chip.fits -x $i
+      keyword mkc EXTNAME extname
+      echo cell $extname
+
+      # skip the video cells
+      rd imci o4743g0247o.$chip.fits -x $i -head
+      keyword imci NAXIS2 tNy
+      if ($tNy > 2000) 
+        echo "skipping video cell $i"
+        wd mkc $outroot.$chip.fits -extend
+      else     
+
+        rd imci o4743g0247o.$chip.fits -x $i
+        $Nx = 590
+        $Ny = 598
+        
+        delete -q imc
+        extract imci imc 0 0 590 598 0 0 590 598
+        
+        # cheat: don't use DATASEC, NAXIS1, NAXIS2: compressed image header is bad
+        set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+        
+        dev -n 1
+        tvch 1; tv mkc -1 10
+        tvch 2; tv imc {$F0-2*$dF} {4*$dF}
+        tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+        resize -n 1 745 610
+        center {590/2} {598/2}
+        
+        $done = 0
+        while ($done == 0)
+          echo $extname
+          echo "y - accept, z - zap, Escape - exit without accepting"
+          cursor 1
+          if ("$KEY" == "y") 
+            $done = 1
+          end
+          if ("$KEY" == "Escape") 
+            break
+          end
+          if ("$KEY" == "z") 
+        	  echo "mark region with 1 & 2"
+            $donezap = 0
+            $skipzap = 0
+        	  while ($donezap == 0)
+              cursor 1
+              if ("$KEY" == "h")
+        	      echo "1 - LL corner"
+        	      echo "2 - UR corner"
+        	      echo "i - U row"
+        	      echo "m - L row"
+        	      echo "j - L col"
+        	      echo "k - R col"
+        	      echo "I - full top"
+        	      echo "J - full left"
+        	      echo "K - full right"
+        	      echo "M - full bottom"
+        	      echo "C - full cell"
+        	      echo "q - stop zapping"
+        	      echo "r - full row"
+        	      echo "c - full col"
+        	      echo "d - left side, full col"
+        	      echo "f - right side, full col"
+              end
+              if ("$KEY" == "1")
+                $xs = int($X1)
+                $ys = int($Y1)
+              end
+              if ("$KEY" == "2")
+                $xe = int($X2)
+                $ye = int($Y2)
+              end
+              if ("$KEY" == "i")
+                $ys = int($Yi)
+              end
+              if ("$KEY" == "m")
+                $ye = int($Ym)
+              end
+              if ("$KEY" == "j")
+                $xs = int($Xj)
+              end
+              if ("$KEY" == "k")
+                $xe = int($Xk)
+              end
+              if ("$KEY" == "I")
+                $ys = 0
+              end
+              if ("$KEY" == "M")
+                $ye = 598
+              end
+              if ("$KEY" == "J")
+                $xs = 0
+              end
+              if ("$KEY" == "K")
+                $xe = 590
+              end
+              if ("$KEY" == "C")
+        	      $xs = 0
+        	      $xe = 590
+        	      $ys = 0
+        	      $ye = 598
+              end
+              if ("$KEY" == "q")
+                $donezap = 1
+              end
+              if ("$KEY" == "Escape")
+                $skipzap = 1
+              end
+              if ("$KEY" == "d")
+                $ys = 0
+        	      $ye = $Ny
+                $xs = int($Xd)
+              end
+              if ("$KEY" == "f")
+                $ys = 0
+        	      $ye = $Ny
+                $xe = int($Xf)
+              end
+              if ("$KEY" == "r")
+                $xs = 0
+                $ys = 6
+        	      $xe = $Nx
+        	      $ye = 8
+                $donezap = 1
+              end
+              if ("$KEY" == "c")
+                $xs = 0
+                $ys = 589
+        	      $xe = $Nx
+        	      $ye = 591
+                $donezap = 1
+              end
+            end            
+            if ($skipzap == 0)
+              zap mkc $xs $ys {$xe-$xs} {$ye-$ys} -v 2
+        	    set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+        	    tvch 1; tv mkc -1 10
+        	    tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+            end
+          end
+          if ("$KEY" == "p") 
+        	  # clip high values
+        	  set mkcp = 2*(imkc > 1e5)
+        	  set mkc = mkc | mkcp
+        	  set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+        	  tvch 1; tv mkc -1 10
+        	  tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+          end
+        end
+        wd mkc $outroot.$chip.fits -extend
+      end
+    end
+  end
+  exec cp -i $outroot.$chip.fits $outroot.$chip.fits.save
+end
+
+# XXX this script has been disabled for write: uncommend the lines marked WWW
+
+macro fixmask
+  if (($0 != 2) && ($0 != 3))
+    echo "USAGE: fixmask (chip) [cell]"
+    echo "subtract 2 from last line number for next cell"
+    break
+  end
+
+  dev -n 0
+  dev -n 1
+  sleep 1
+
+  dev -n 0
+  showchips $1
+  $outroot = GPC1.MASK.FIX
+  $chip = $1
+
+  $startcell = -1
+  if ($0 == 3)
+    $startcell = $2
+  end
+
+  for i $startcell 64
+    if ($i == -1)
+      rd mkc GPC1.MASK.$chip.fits
+      keyword mkc NEXTEND -wd 0
+      # WWW wd mkc $outroot.$chip.fits
+      continue
+    else
+      rd mkc GPC1.MASK.$chip.fits -x $i
+      keyword mkc EXTNAME extname
+      echo cell $extname
+
+      rd imci o4596g0022f.13750.detproc.13.$chip.fits -x $i
+
+      delete -q imc
+      extract imci imc 0 0 590 598 0 0 590 598
+
+      # cheat: don't use DATASEC, NAXIS1, NAXIS2: compressed image header is bad
+      # window down -5,-5,-5,-5
+      # datasec 1:590,1:598
+      # keyword mkc NAXIS1 Nx
+      # keyword mkc NAXIS2 Ny
+      $Nx = 590
+      $Ny = 598
+      zap mkc 0 0 6 $Ny -v 2
+      zap mkc 0 0 $Nx 6 -v 2
+      zap mkc 583 0 7 $Ny -v 2
+      zap mkc 0 591 $Nx 7 -v 2
+
+      set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+
+      dev -n 1
+      tvch 1; tv mkc -1 10
+      tvch 2; tv imc {$F0-2*$dF} {4*$dF}
+      tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+
+      $done = 0
+      while ($done == 0)
+        echo $extname
+        echo "y - accept, z - zap"
+        cursor 1
+        if ("$KEY" == "y") 
+          $done = 1
+        end
+        if ("$KEY" == "Escape") 
+          break
+        end
+        if ("$KEY" == "z") 
+	  echo "mark region with 1 & 2"
+          $donezap = 0
+          $skipzap = 0
+ 	  while ($donezap == 0)
+            cursor 1
+            if ("$KEY" == "1")
+              $xs = int($X1)
+              $ys = int($Y1)
+            end
+            if ("$KEY" == "2")
+              $xe = int($X2)
+              $ye = int($Y2)
+            end
+            if ("$KEY" == "i")
+              $ys = int($Yi)
+            end
+            if ("$KEY" == "m")
+              $ye = int($Ym)
+            end
+            if ("$KEY" == "j")
+              $xs = int($Xj)
+            end
+            if ("$KEY" == "k")
+              $xe = int($Xk)
+            end
+            if ("$KEY" == "q")
+              $donezap = 1
+            end
+            if ("$KEY" == "Escape")
+              $skipzap = 1
+            end
+            if ("$KEY" == "d")
+              $ys = 0
+	      $ye = $Ny
+              $xs = int($Xd)
+            end
+            if ("$KEY" == "f")
+              $ys = 0
+	      $ye = $Ny
+              $xe = int($Xf)
+            end
+            if ("$KEY" == "r")
+              $xs = 0
+              $ys = 6
+	      $xe = $Nx
+	      $ye = 8
+              $donezap = 1
+            end
+            if ("$KEY" == "c")
+              $xs = 0
+              $ys = 589
+	      $xe = $Nx
+	      $ye = 591
+              $donezap = 1
+            end
+          end            
+          if ($skipzap == 0)
+            zap mkc $xs $ys {$xe-$xs} {$ye-$ys} -v 2
+   	    set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+      	    tvch 1; tv mkc -1 10
+      	    tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+          end
+        end
+        if ("$KEY" == "p") 
+	  # clip high values
+	  set mkcp = 2*(imkc > 1e5)
+	  set mkc = mkc | mkcp
+ 	  set imkc = (imc*(abs(imc - $F0) < $dF) + 1e6*(abs(imc - $F0) > $dF))*(mkc < 1)
+      	  tvch 1; tv mkc -1 10
+      	  tvch 3; tv imkc {$F0-2*$dF} {4*$dF}
+        end
+      end
+      # WWW wd mkc $outroot.$chip.fits -extend
+    end
+  end
+  exec cp -i $outroot.$chip.fits $outroot.$chip.fits.save
+end
+
+# $F0 = 16000
+# $F0 = 20500
+$F0 = 390
+$dF = 200
+macro showchips
+  if ($0 != 2)
+    echo "USAGE: showchips (chip)"
+    break
+  end
+
+  $chip = $1
+  rd mk GPC1.MASK.$chip.ch.fits -x 0
+  rd im o4743g0247o.$chip.ch.fits -x 0
+  stats im
+  $F0 = $MEDIAN
+  set imk = (im*(abs(im - $F0) < $dF) + 1e6*(abs(im - $F0) > $dF))*(mk < 10)
+  tvch 1; tv mk 0 10
+  tvch 2; tv im {$F0-2*$dF} {4*$dF}
+  tvch 3; tv imk {$F0-2*$dF} {4*$dF}
+end
+
+macro go
+  for i 0 8
+   for j 0 8
+    if (($i == 0) && ($j == 0)) continue
+    if (($i == 0) && ($j == 7)) continue
+    if (($i == 7) && ($j == 0)) continue
+    if (($i == 7) && ($j == 7)) continue
+    showchips XY\$i\$j
+    cursor
+   end
+  end
+end
+
+# XY43 was written with cell xy26 multiple times (same EXTNAME)
+macro fix.XY43
+  $input = GPC1.MASK.20081109.XY43.fits.broken
+  $output = GPC1.MASK.20081109.XY43.fits.fix
+
+  # PHU
+  rd mk $input
+  keyword mk NEXTEND -wd 0
+  wd mk $output
+
+  # EXT
+  for i 0 66
+    if ($i == 22) continue
+    if ($i == 23) continue
+    rd mk $input -x $i
+    wd mk $output -extend
+  end
+end
+
+# XY43 was written with cell xy26 multiple times (same EXTNAME)
+macro fix.headers
+  if ($0 != 2)
+    echo "USAGE: fix.headers (chip)"
+    break
+  end
+
+  $input = GPC1.MASK.20081109.$1.fits.broken
+  $output = GPC1.MASK.20081109.$1.fits.fix
+
+  # PHU
+  rd mk $input
+  keyword mk NEXTEND -wd 0
+
+  keyword -d mk AIRMASS
+  keyword -d mk FILTERID
+  keyword -d mk POSANGLE
+  keyword -d mk RADECSYS
+  keyword -d mk COMRA   
+  keyword -d mk COMDEC  
+  keyword -d mk OBSTYPE 
+  keyword -d mk OBJECT  
+  keyword -d mk ALT     
+  keyword -d mk AZ      
+  keyword -d mk MJD-OBS 
+  keyword -d mk DETTEM  
+  keyword -d mk M1X     
+  keyword -d mk M1Y     
+  keyword -d mk M1Z     
+  keyword -d mk M1TIP   
+  keyword -d mk M1TILT  
+  keyword -d mk M2X     
+  keyword -d mk M2Y     
+  keyword -d mk M2Z     
+  keyword -d mk M2TIP   
+  keyword -d mk M2TILT  
+  keyword -d mk ENVTEM  
+  keyword -d mk ENVHUM  
+  keyword -d mk ENVWIN  
+  keyword -d mk ENVDIR  
+  keyword -d mk TELTEMM1
+  keyword -d mk TELTEMMS
+  keyword -d mk TELTEMM2
+  keyword -d mk TELTEMSP
+  keyword -d mk TELTEMTR
+  keyword -d mk TELTEMEX
+  keyword -d mk PONTIME 
+  keyword -d mk EXPREQ  
+
+  date -var date
+  keyword mk DATE -w "$date"
+  keyword mk DATE -wc "Date of File Creation"
+  keyword mk COMMENT -ws "Mask Image Generated for IPP by EAM"
+
+  keyword mk ORIGIN   -w "PS1"               ; keyword mk ORIGIN   -wc "IfA Pan-STARRS 1"
+  keyword mk INSTRUME -w "gpc1"              ; keyword mk INSTRUME -wc "Instrument Name"                               
+  keyword mk IMNAXIS1 -wd               4846 ; keyword mk IMNAXIS1 -wc "OTA image width in unbinned pixels"            
+  keyword mk IMNAXIS2 -wd               4868 ; keyword mk IMNAXIS2 -wc "OTA image height in unbinned pixels"           
+  keyword mk IMTM1_1  -wf        -1.00000000 ; keyword mk IMTM1_1  -wc "IMage Transform Matrix. A unit vector in the"  
+  keyword mk IMTM1_2  -wf         0.00000000 ; keyword mk IMTM1_2  -wc " serial (fast) direction of amplifier readout" 
+  keyword mk IMTM2_1  -wf         0.00000000 ; keyword mk IMTM2_1  -wc "And a unit vector in parallel (slow) direction"
+  keyword mk IMTM2_2  -wf         1.00000000 ; keyword mk IMTM2_2  -wc " of amplifier readout"                         
+  keyword mk CELLGAP1 -wd                 18 ; keyword mk CELLGAP1 -wc "Cell gap between columns (in unbinned pixels") 
+  keyword mk CELLGAP2 -wd                 12 ; keyword mk CELLGAP2 -wc "Cell gap between rows (in unbinned pixels")    
+  keyword mk CNAXIS1  -wd                590 ; keyword mk CNAXIS1  -wc "Cell imaging size in unbinned pixel columns"   
+  keyword mk CNAXIS2  -wd                598 ; keyword mk CNAXIS2  -wc "Cell imaging size in unbinned pixel rows"      
+  keyword mk CNPIX1   -wd                  0 ; keyword mk CNPIX1   -wc "Offset in unbinned pixel cols to readout start"
+  keyword mk CNPIX2   -wd                  0 ; keyword mk CNPIX2   -wc "Offset in unbinned pixel rows to readout start"
+  keyword mk CCDBIN1  -wd                  1 ; keyword mk CCDBIN1  -wc "Binning factor along axis 1"                   
+  keyword mk CCDBIN2  -wd                  1 ; keyword mk CCDBIN2  -wc "Binning factor along axis 2"                   
+  keyword mk PRESCAN1 -wd                  0 ; keyword mk PRESCAN1 -wc "Prescan count on axis 1"                       
+  keyword mk PRESCAN2 -wd                  0 ; keyword mk PRESCAN2 -wc "Prescan count on axis 2"                       
+  keyword mk OVRSCAN1 -wd                 34 ; keyword mk OVRSCAN1 -wc "Overscan count on axis 1"                      
+  keyword mk OVRSCAN2 -wd                 10 ; keyword mk OVRSCAN2 -wc "Overscan count on axis 2"                      
+  keyword mk COMMENT  -ws ""
+  keyword mk COMMENT  -ws " Iraf coordinates "
+  keyword mk COMMENT  -ws " ---------------- "
+  keyword mk CCDSUM   -w  "1 1"              ; keyword mk CCDSUM   -wc "Binning factors"
+  keyword mk DETSIZE  -w  "[1:4846,1:4868]"  ; keyword mk DETSIZE  -wc "Iraf total image pixels in full mosaic"
+
+  wd mk $output
+
+  # EXT
+  for i 0 64
+    rd mk $input -x $i
+
+# delete these:
+    keyword mk -d CHECKSUM
+    keyword mk -d DATASUM
+    keyword mk -d EXPTIME
+    keyword mk -d DARKTIME
+
+# we currently have:
+# OK: DATASEC = '[1:590,1:598]'      / Trim section                                  
+# OK: LTM1_1  =                   -1 / Orientation in x compared to the rest of the c
+# OK: LTM2_2  =                    1 / Orientation in y compared to the rest of the c
+# OK: CCDSUM  = '1 1     '           / Binning in x                                  
+# OK: IMNPIX1 =                  590 / Position of (0,0) on the chip                 
+# OK: IMNPIX2 =                  610 / Position of (0,0) on the chip                 
+
+# insert these as is
+    keyword mk LTM1_1 LT11
+    keyword mk LTM2_2 LT22
+
+    keyword mk CCDSUM   -w "1 1"                ; keyword mk CCDSUM  -wc "Binning factors"
+    keyword mk BIASSEC  -w "[0:0,0:0]"          ; keyword mk BIASSEC -wc "Overscan (bias) area"
+    keyword mk DETSIZE  -w "[1:4846,1:4868]"    ; keyword mk DETSIZE -wc "Iraf total image pixels in full mosaic"
+    keyword mk ATM1_1   -wf $LT11               ; keyword mk ATM1_1  -wc "Iraf amplifier transformation matrix"
+    keyword mk ATM2_2   -wf $LT22               ; keyword mk ATM2_2  -wc "Iraf amplifier transformation matrix"
+
+# need to get the parity of these right:
+# DETSEC  = '[590:1,1:598]'      / Iraf mosaic area of the detector              
+# DETSEC  = '[590:1,611:1208]'   / Iraf mosaic area of the detector              
+# DETSEC  = '[IMNPIX1:IMNPIX1-NAXIS1+1,IMNPIX2+1:IMNPIX2+NAXIS2]'
+# DETSEC  = '[IMNPIX1:IMNPIX1-NAXIS1+1,IMNPIX2+1:IMNPIX2+NAXIS2]'
+# LTV1    =         591.00000000 / Iraf image transformation vector              
+# LTV2    =        -610.00000000 / Iraf image transformation vector              
+# ATV1    =         591.00000000 / Iraf amplifier transformation vector          
+# ATV2    =        -610.00000000 / Iraf amplifier transformation vector          
+
+    wd mk $output -extend
+  end
+end
+
+macro mask.fraction
+
+  # make an output images which has 4x4 pixels per cell (256x256)
+  mcreate out 256 256
+
+  $CameraMasked = 0
+  $CameraAllPix = 0
+
+  for ix 0 8
+    for iy 0 8
+      if (($ix == 0) && ($iy == 0)) continue
+      if (($ix == 0) && ($iy == 7)) continue
+      if (($ix == 7) && ($iy == 0)) continue
+      if (($ix == 7) && ($iy == 7)) continue
+
+      sprintf file GPC1.MASK.FIX.XY%d%d.fits $ix $iy
+      for Ix 0 8
+        for Iy 0 8
+          sprintf cell xy%d%d $Ix $Iy
+	  rd mk $file -n $cell
+          clip mk 0 0 1 1
+          stats -q mk
+          $CameraMasked = $CameraMasked + $TOTAL
+          $CameraAllPix = $CameraAllPix + $NPIX
+          $frac = $TOTAL / $NPIX
+          if ($ix < 4)
+            zap out {4*(8*($ix + 1) - $Ix - 1)} {4*(8*($iy + 1) - $Iy - 1)} 4 4 -v $frac
+          else
+            zap out {4*(8*$ix + $Ix)} {4*(8*$iy + $Iy)} 4 4 -v $frac
+          end
+        end
+      end
+    end
+  end
+
+  wd out fraction.fits -bitpix -32 -bzero 0.0 -bscale 1.0
+end
+
+macro mask.fraction.cte
+
+  # make an output images which has 4x4 pixels per cell (256x256)
+  mcreate out 256 256
+
+  $CameraMasked = 0
+  $CameraAllPix = 0
+
+  for ix 0 8
+    for iy 0 8
+      if (($ix == 0) && ($iy == 0)) continue
+      if (($ix == 0) && ($iy == 7)) continue
+      if (($ix == 7) && ($iy == 0)) continue
+      if (($ix == 7) && ($iy == 7)) continue
+
+      sprintf file GPC1.MASK.CTE.XY%d%d.fits $ix $iy
+      for Ix 0 8
+        for Iy 0 8
+          sprintf cell xy%d%d $Ix $Iy
+	  rd mk $file -n $cell
+          clip mk 0 0 1 1
+          stats -q mk
+          $CameraMasked = $CameraMasked + $TOTAL
+          $CameraAllPix = $CameraAllPix + $NPIX
+          $frac = $TOTAL / $NPIX
+          if ($ix < 4)
+            zap out {4*(8*($ix + 1) - $Ix - 1)} {4*(8*($iy + 1) - $Iy - 1)} 4 4 -v $frac
+          else
+            zap out {4*(8*$ix + $Ix)} {4*(8*$iy + $Iy)} 4 4 -v $frac
+          end
+        end
+      end
+    end
+  end
+
+  wd out fraction.cte.fits -bitpix -32 -bzero 0.0 -bscale 1.0
+end
+
+macro pics
+
+  resize 710 560
+  tv out 0.0 1.0
+  tvcolors rainbow
+  point red CIRCLE 128 128 133 133
+  jpeg -name mask-full-focalplane.jpg
+
+  set x = xramp(out)
+  set y = yramp(out)
+  set r = sqrt((x-128)^2 + (y-128)^2)
+  set rm = (r < 133)
+  set rf = out * rm
+  tv rf
+  jpeg -name mask-unvignetted.jpg
+  
+  stats -q out
+  fprintf "masked fraction (full focal plane): %.3f" {$TOTAL/(4^2*8^2*60)}
+
+  stats -q rf
+  fprintf "masked fraction (unvignetted region): %.3f" {$TOTAL/(3.14*133^2)}
+
+end
+
+macro pics.cte
+
+  rd out fraction.cte.fits
+  resize 710 560
+  tv out 0.0 1.0
+  tvcolors rainbow
+  point red CIRCLE 128 128 133 133
+  jpeg -name mask-full-focalplane.cte.jpg
+
+  set x = xramp(out)
+  set y = yramp(out)
+  set r = sqrt((x-128)^2 + (y-128)^2)
+  set rm = (r < 133)
+  set rf = out * rm
+  tv rf
+  jpeg -name mask-unvignetted.cte.jpg
+  
+  stats -q out
+  fprintf "masked fraction (full focal plane): %.3f" {$TOTAL/(4^2*8^2*60)}
+
+  stats -q rf
+  fprintf "masked fraction (unvignetted region): %.3f" {$TOTAL/(3.14*133^2)}
+
+end
Index: branches/bills_081204/tools/mask_extra.c
===================================================================
--- branches/bills_081204/tools/mask_extra.c	(revision 20890)
+++ branches/bills_081204/tools/mask_extra.c	(revision 20890)
@@ -0,0 +1,104 @@
+// Use a mask description (which can be generated from a ds9 region file using ds9_regions_mask.pl)
+// to mask extra pixels.
+
+// Compilation: gcc mask_extra.c -o mask_extra -std=gnu99 -g -Wall `pslib-config --cflags --libs`
+// Usage: mask_extra OLD_MASK.fits MASK_DESCRIPTION.dat NEW_MASK.fits
+
+#include <stdio.h>
+#include <pslib.h>
+
+static bool readImage(psImage **image, psMetadata **header, const char *name)
+{
+    psFits *fits = psFitsOpen(name, "r");
+    if (header) {
+        *header = psFitsReadHeader(NULL, fits);
+    }
+    if (image) {
+        *image = psFitsReadImage(fits, psRegionSet(0,0,0,0), 0);
+    }
+    psFitsClose(fits);
+    return true;
+}
+
+static void writeImage(const psImage *image,   // Image to write
+                       psMetadata *header, // Header to write
+                       const char *name        // Name of FITS file
+    )
+{
+    psFits *fits = psFitsOpen(name, "w");
+    psFitsWriteImage(fits, header, image, 0, NULL);
+    psFitsClose(fits);
+}
+
+int main(int argc, char *argv[])
+{
+    if (argc != 4) {
+        fprintf(stderr, "Not enough arguments\n\n");
+        exit(EXIT_FAILURE);
+    }
+
+    const char *origName = argv[1];     // Original mask name
+    const char *descName = argv[2];     // Mask description file name
+    const char *outName = argv[3];      // Output file name
+
+    psImage *mask;                      // Mask image
+    psMetadata *header;                 // Header for image
+
+    readImage(&mask, &header, origName);
+    int numCols = mask->numCols, numRows = mask->numRows; // Size of image
+
+    psArray *desc = psVectorsReadFromFile(descName, "%d %f %f %f"); // Mask description
+    psVector *type = desc->data[0];     // Type of mask
+    psVector *x = desc->data[1];        // x coordinates
+    psVector *y = desc->data[2];        // y coordinates
+    psVector *rad = desc->data[3];      // Radius
+    int num = type->n;                  // Number of masks
+
+    int maxType = 0;                    // Maximum number of types
+    for (int i = 0; i < num; i++) {
+        if (type->data.S32[i] > maxType) {
+            maxType = type->data.S32[i];
+        }
+    }
+    psVector *count = psVectorAlloc(maxType + 1, PS_TYPE_S32); // Count of each type
+    psVectorInit(count, 0);
+
+    for (int i = 0; i < num; i++) {
+        float xCen = x->data.F32[i] - 1, yCen = y->data.F32[i] - 1; // Coordinates of centre
+        float radius = rad->data.F32[i];// Radius of circle
+        float radius2 = PS_SQR(radius); // Radius of circle, squared
+
+        // Bounds of circle
+        int xMin = PS_MAX(xCen - radius, 0), xMax = PS_MIN(xCen + radius, numCols - 1);
+        int yMin = PS_MAX(yCen - radius, 0), yMax = PS_MIN(yCen + radius, numRows - 1);
+
+        int numMask = 0;                // Number of pixels masked
+        for (int v = yMin; v <= yMax; v++) {
+            int v2 = PS_SQR(v - yCen);  // Distance
+            for (int u = xMin; u <= xMax; u++) {
+                if (PS_SQR(u - xCen) + v2 > radius2) {
+                    continue;
+                }
+                mask->data.PS_TYPE_MASK_DATA[v][u] = 0xFF;
+                numMask++;
+            }
+        }
+        count->data.S32[type->data.S32[i]] += numMask;
+    }
+
+    psFree(desc);
+
+    long numPix = numCols * numRows;    // Number of pixels
+    for (int i = 0; i <= maxType; i++) {
+        printf("Type %d: %d (%f%%)\n", i, count->data.S32[i], count->data.S32[i] / (float)numPix * 100.0);
+    }
+
+    psFree(count);
+
+    writeImage(mask, header, outName);
+
+    psFree(mask);
+    psFree(header);
+
+    return PS_EXIT_SUCCESS;
+}
Index: branches/bills_081204/tools/stack_inputs.pl
===================================================================
--- branches/cnb_branch_20081104/tools/stack_inputs.pl	(revision 20530)
+++ branches/bills_081204/tools/stack_inputs.pl	(revision 20890)
@@ -71,8 +71,8 @@
     foreach my $type ( keys %{EXTENSIONS()} ) {
         my $ext = ${EXTENSIONS()}{$type}; # Extension
-        my $neb = "$warp_path$ext"; # Nebulous file
-        my $source = `neb-locate --path $neb` or die "Unable to locate $neb\n"; # Actual file
+        my $file = "$warp_path$ext"; # File
+        my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Resolved file name
         chomp $source;
-        my ($target) = $source =~ m|^.*:(.*)|; # Target name
+        my ($target) = $source =~ m|^.*[:/](.*)|; # Target name
         !system "cp $source $target" or die "Unable to copy $source\n";
         print $list "\t$type\tSTR\t$target\n" if defined $input;
Index: branches/bills_081204/tools/stack_outputs.pl
===================================================================
--- branches/cnb_branch_20081104/tools/stack_outputs.pl	(revision 20530)
+++ branches/bills_081204/tools/stack_outputs.pl	(revision 20890)
@@ -16,4 +16,7 @@
                              'WEIGHT' => '.wt.fits', # Weight
                              'SOURCES' => '.cmf', # Sources
+			     'LOG' => '.log', # Log file
+			     'B1' => '.b1.jpg',	# Binned jpeg
+			     'B2' => '.b2.jpg',	# Binned jpeg
                          };
 
Index: branches/bills_081204/tools/warp_inputs.pl
===================================================================
--- branches/bills_081204/tools/warp_inputs.pl	(revision 20890)
+++ branches/bills_081204/tools/warp_inputs.pl	(revision 20890)
@@ -0,0 +1,114 @@
+#!/usr/bin/env perl
+
+#
+# Copy warp inputs to the current directory
+#
+
+use warnings;
+use strict;
+
+use DBI;
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+use constant IMAGE => '.ch.fits';  # Image extension
+use constant MASK => '.ch.mk.fits'; # Mask extension
+use constant WEIGHT => '.ch.wt.fits'; # Weight extension
+use constant ASTROM => '.smf';  # Astrometry extension
+
+
+
+my ($db_host, $db_name, $db_user, $db_pw); # Database details
+my ($warp_id);                  # Warp of interest
+my ($skycell_id);               # Skycell of interest
+my ($image, $mask, $weight); # Files to contain input lists
+
+GetOptions(
+           'dbhost=s' => \$db_host, # Database host name
+           'dbname=s' => \$db_name, # Database name
+           'dbuser=s' => \$db_user, # Database user
+           'dbpass=s' => \$db_pw, # Database p/w
+           'warp_id=s' => \$warp_id, # Warp identifier
+           'skycell_id=s' => \$skycell_id, # Skycell identifier
+           'image=s' => \$image, # File with images
+           'mask=s' => \$mask,  # File with masks
+           'weight=s' => \$weight, # File with weights
+           ) or die "Unable to parse arguments.\n";
+die "Unknown option: @ARGV\n" if @ARGV;
+die "Required options: --dbhost --dbname --dbuser --dbpass --warp_id\n"
+    unless defined $db_host
+    and defined $db_name
+    and defined $db_user
+    and defined $db_pw
+    and defined $warp_id
+    and defined $skycell_id;
+
+# Database connection
+my $db = DBI->connect( "DBI:mysql:database=$db_name;host=$db_host;mysql_socket=" . DB_SOCKET(),
+                       $db_user,
+                       $db_pw,
+                       { RaiseError => 1, AutoCommit => 1 }
+                       ) or die "Unable to connect to database: $DBI::errstr";
+
+# Query to run
+my $sql = "SELECT" .
+    " chipProcessedImfile.path_base AS chip_path_base," .
+    " class_id," .
+    " camProcessedExp.path_base AS cam_path_base" .
+    " FROM warpRun" .
+    " JOIN warpSkyCellMap USING(warp_id)" .
+    " JOIN fakeRun USING(fake_id)" .
+    " JOIN camRun USING(cam_id)" .
+    " JOIN camProcessedExp USING(cam_id)" .
+    " JOIN chipProcessedImfile USING(chip_id, class_id)" .
+    " WHERE warp_id = $warp_id" .
+    " AND skycell_id = \'$skycell_id\'";
+
+
+my $results = $db->selectall_arrayref( $sql ) or die "Unable to execute SQL: $DBI::errstr";
+$db->disconnect;
+
+my ($imageList, $maskList, $weightList); # Files to write to
+open $imageList, "> $image" if defined $image;
+open $maskList, "> $mask" if defined $mask;
+open $weightList, "> $weight" if defined $weight;
+
+foreach my $row ( @$results ) {
+    my $chip = $$row[0];        # Chip path
+    my $class_id = $$row[1];    # Class identifier
+    my $cam = $$row[2];         # Camera path
+
+    $chip .= '.' . $class_id;
+
+    my $imageName = $chip . IMAGE(); # Name of image
+    my $maskName = $chip . MASK(); # Name of mask
+    my $weightName = $chip . WEIGHT(); # Name of weight
+    my $astromName = $cam . ASTROM(); # Name of astrometry
+
+    print $imageList "$imageName\n" if defined $image;
+    print $maskList "$maskName\n" if defined $mask;
+    print $weightList "$weightName\n" if defined $weight;
+
+    copy_file( $imageName );
+    copy_file( $maskName );
+    copy_file( $weightName );
+    copy_file( $astromName );
+}
+
+close $imageList if defined $image;
+close $maskList if defined $mask;
+close $weightList if defined $weight;
+
+
+# Pau.
+
+
+sub copy_file
+{
+    my $file = shift;           # File to copy
+
+    my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file
+    chomp $source;
+    my ($target) = $source =~ m|^.*[:/](.*)|; # Target name
+    !system "cp $source $target" or die "Unable to copy $source\n";
+}
Index: branches/bills_081204/tools/warp_outputs.pl
===================================================================
--- branches/bills_081204/tools/warp_outputs.pl	(revision 20890)
+++ branches/bills_081204/tools/warp_outputs.pl	(revision 20890)
@@ -0,0 +1,85 @@
+#!/usr/bin/env perl
+
+#
+# Copy warp outputs to the current directory
+#
+
+use warnings;
+use strict;
+
+use DBI;
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+
+use constant EXTENSIONS => { 'IMAGE' => '.fits', # Image
+                             'MASK' => '.mask.fits', # Mask
+                             'WEIGHT' => '.wt.fits', # Weight
+                             'SOURCES' => '.cmf', # Sources
+                         };
+
+my ($db_host, $db_name, $db_user, $db_pw); # Database details
+my ($warp_id);                  # Warp of interest
+my ($skycell_id);               # Skycell of interest
+my ($input);                    # Input list
+my ($products);                 # Products of interest
+
+GetOptions(
+           'dbhost=s' => \$db_host, # Database host name
+           'dbname=s' => \$db_name, # Database name
+           'dbuser=s' => \$db_user, # Database user
+           'dbpass=s' => \$db_pw, # Database p/w
+           'warp_id=s' => \$warp_id, # Warp identifier
+           'skycell_id=s' => \$skycell_id, # Skycell identifier
+           'products=s' => \$products, # Products of interest
+           ) or die "Unable to parse arguments.\n";
+die "Unknown option: @ARGV\n" if @ARGV;
+die "Required options: --dbhost --dbname --dbuser --dbpass --warp_id\n"
+    unless defined $db_host
+    and defined $db_name
+    and defined $db_user
+    and defined $db_pw
+    and defined $warp_id;
+
+# Database connection
+my $db = DBI->connect( "DBI:mysql:database=$db_name;host=$db_host;mysql_socket=" . DB_SOCKET(),
+                       $db_user,
+                       $db_pw,
+                       { RaiseError => 1, AutoCommit => 1 }
+                       ) or die "Unable to connect to database: $DBI::errstr";
+
+# Query to run
+my $sql = "SELECT path_base FROM warpSkyfile WHERE warp_id = $warp_id AND ignored = 0";
+$sql .= " AND skycell_id = \'$skycell_id\'" if defined $skycell_id;
+
+# List of diffs
+my $warps = $db->selectcol_arrayref( $sql,
+                                     { Columns => [1] }
+                                     ) or die "Unable to execute SQL: $DBI::errstr";
+
+my @products;                   # Array of products
+if (defined $products) {
+    @products = split /,/, $products;
+} else {
+    @products = keys %{EXTENSIONS()};
+}
+
+foreach my $warp ( @$warps ) {
+    foreach my $product ( @products ) {
+        copy_extension( $warp, ${EXTENSIONS()}{$product} );
+    }
+}
+
+# Pau.
+
+
+sub copy_extension
+{
+    my $path = shift;           # Path to copy
+    my $ext = shift;            # Extension to copy
+
+    my $file = "$path$ext"; # File
+    my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file
+    chomp $source;
+    my ($target) = $source =~ m|^.*[:/](.*)|; # Target name
+    !system "cp $source $target" or die "Unable to copy $source\n";
+}