Index: branches/pap/magic/remove/src/streaksio.c
===================================================================
--- branches/pap/magic/remove/src/streaksio.c	(revision 23948)
+++ branches/pap/magic/remove/src/streaksio.c	(revision 25027)
@@ -10,8 +10,13 @@
 static nebServer *ourNebServer = NULL;
 
+// Assumptions about the file structure of non-raw files
+// The 'image' for each file (image, mask weight) is contained in the first
+// extension. 
+
+
 // open the files required for streaks procesing.
 // if remove is true the calling program is streaksremove and the recovery files are outputs
 // if false the recovery files are inputs
-streakFiles *openFiles(pmConfig *config, bool remove)
+streakFiles *openFiles(pmConfig *config, bool remove, char *program_name)
 {
     bool status;
@@ -21,4 +26,11 @@
 
     sf->config = config;
+    sf->program_name = basename(program_name);
+
+    if (remove) {
+        // remember pointer so that streaksExit can delete temps
+        setStreakFiles(sf);
+    }
+
 
     // error checking is done by sFileOpen. If a file can't be opened we just exit
@@ -59,9 +71,12 @@
     // if we are passed a mask it is camera stage mask and we also
     // need to destreak the chip level mask file as well
+    // If it doesn't exist, we didn't have a camera mask
     if (remove && sf->inMask && (stage == IPP_STAGE_CHIP)) {
-        sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, true);
-        inputBasename = basename(sf->inChMask->name);
-        sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
-        sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+        sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false);
+        if (sf->inChMask) {
+            inputBasename = basename(sf->inChMask->name);
+            sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+            sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
+        }
     }
 
@@ -74,4 +89,11 @@
         } else {
             sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true);
+        }
+    }
+    if (remove) {
+        sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false);
+        if (sf->inSources) {
+            inputBasename = basename(sf->inSources->name);
+            sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
         }
     }
@@ -159,4 +181,33 @@
 }
 
+// figure out if a nebulous instance is a non-destreaked file
+static bool
+nebFileIsDestreaked(sFile *sfile)
+{
+    if (!sfile->resolved_name) {
+        psError(PS_ERR_PROGRAMMING, true, "resolved name is null");
+        return false;
+    }
+    psFits *fits = psFitsOpen(sfile->resolved_name, "r");
+    if (!fits) {
+        psError(PS_ERR_IO, true, "failed open %s", sfile->name);
+        // can't tell if it is a destreaked file
+        return false;
+    }
+    psMetadata *header = psFitsReadHeader(NULL, fits);
+    if (!header) {
+        psError(PS_ERR_IO, true, "failed to read header for: %s", sfile->name);
+        return false;
+    }
+    bool mdok;
+    bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK");
+    if (mdok && isDestreaked) {
+        return true;
+    } else {
+        psError(PS_ERR_IO, false, "output file already exists and may not be de-streaked: %s", sfile->name);
+        return false;
+    }
+}
+
 static psString
 resolveFilename(pmConfig *config, sFile *sfile, bool create)
@@ -170,5 +221,11 @@
             // delete the existing file, since there may be more than one
             // instance. It will get created below in pmConfigConvertFilename
-            if (nebFind(server, sfile->name)) {
+            if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) {
+                if (!nebFileIsDestreaked(sfile)) {
+                    psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name);
+                    return NULL;
+                }
+                nebFree(sfile->resolved_name);
+                sfile->resolved_name = NULL;
                 nebDelete(server, sfile->name);
             }
@@ -192,5 +249,5 @@
     // all of the keywords in the raw image files written to the output destreaked files
 
-    if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
+    if (!outputFilename && !CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
         // stage is warp or diff AND fileSelect eq "INPUT"
         // get data from pmFPAfile.
@@ -234,4 +291,6 @@
         sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header);
 
+        sfile->nHDU = psFitsGetSize(sfile->pmfile->fits);
+
         return sfile;
     }
@@ -249,5 +308,5 @@
     }
 
-    // if outputFilename is not null name it contains the "directory"
+    // if outputFilename is not null name it contains the "directory" (perhaps with a prefix like SR_)
     // and outputFilename is the basename name of the file (or nebulous key)
     // and the file is to be opened for writing
@@ -333,4 +392,18 @@
 
 void
+addDestreakKeyword(psMetadata *header)
+{
+    psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE,
+        "Have streaks been removed from image?", true);
+}
+
+void
+addRecoveryKeyword(psMetadata *header)
+{
+    psMetadataAddBool(header, PS_LIST_TAIL, "PSRECOVR", PS_META_REPLACE,
+        "Does this image contain excised streak pixels?", true);
+}
+
+void
 copyPHU(streakFiles *sfiles, bool remove)
 {
@@ -343,6 +416,13 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-
-    // TODO: add keyword indicating that streaks have been removed
+    psMetadata *recHeader = NULL;
+    if (remove && sfiles->recImage) {
+       recHeader = psMetadataCopy(NULL, imageHeader);
+       addRecoveryKeyword(recHeader);
+    }
+
+    // add keyword indicating that streaks have been removed
+    addDestreakKeyword(imageHeader);
+
     if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
         psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -350,10 +430,11 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-    // TODO: add keyword indicating that this is the recovery image
-    if (remove && sfiles->recImage && !psFitsWriteBlank(sfiles->recImage->fits, imageHeader, NULL)) {
+    if (recHeader && !psFitsWriteBlank(sfiles->recImage->fits, recHeader, NULL)) {
         psError(PS_ERR_IO, false, "failed to write primary header to %s",
             sfiles->recImage->resolved_name);
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
+    psFree(recHeader);
+    recHeader = NULL;
     psFree(imageHeader);
 
@@ -366,5 +447,11 @@
             streaksExit("", 1);
         }
-        // TODO: add keyword indicating that streaks have been removed
+        if (remove && sfiles->recMask) {
+            recHeader = psMetadataCopy(NULL, maskHeader);
+            // add keyword indicating that this is the recovery image
+           addRecoveryKeyword(recHeader);
+        }
+        // add keyword indicating that streaks have been removed
+        addDestreakKeyword(maskHeader);
         if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -372,10 +459,11 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        // TODO: add keyword indicating that this is the recovery image
-        if (remove && sfiles->recMask && !psFitsWriteBlank(sfiles->recMask->fits, maskHeader, NULL)) {
+        if (recHeader && !psFitsWriteBlank(sfiles->recMask->fits, recHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
                 sfiles->recMask->resolved_name);
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
+        psFree(recHeader);
+        recHeader = NULL;
         psFree(maskHeader);
     }
@@ -388,5 +476,12 @@
             streaksExit("", 1);
         }
-        // TODO: add keyword indicating that streaks have been removed
+        if (remove && sfiles->recWeight) {
+            recHeader = psMetadataCopy(NULL, weightHeader);
+            // add keyword indicating that this is a recovery image
+           addRecoveryKeyword(recHeader);
+        }
+
+        // add keyword indicating that streaks have been removed
+        addDestreakKeyword(weightHeader);
         if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -394,6 +489,5 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        // TODO: add keyword indicating that this is a recovery image
-        if (remove && sfiles->recWeight && !psFitsWriteBlank(sfiles->recWeight->fits, weightHeader, NULL)) {
+        if (recHeader && !psFitsWriteBlank(sfiles->recWeight->fits, recHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
                 sfiles->recWeight->resolved_name);
@@ -401,4 +495,5 @@
         }
         psFree(weightHeader);
+        psFree(recHeader);
     }
 }
@@ -530,5 +625,5 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-
+ 
     bool status;
     in->extname = psMetadataLookupStr(&status, in->header, "EXTNAME");
@@ -565,5 +660,6 @@
 
 static void
-setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero)
+setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero, psFitsCompressionType compType,
+    psVector *tiles)
 {
     if (!sfile) {
@@ -578,9 +674,17 @@
     sfile->fits->options->bscale = bscale;
     sfile->fits->options->bzero = bzero;
-}
-
-void
-copyFitsOptions(sFile *out, sFile *rec, sFile *in)
-{
+
+    psFitsSetCompression(sfile->fits, compType, tiles, 8, 0, 0);
+}
+
+void
+copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles)
+{
+    bool mdok;
+    psString compTypeStr = psMetadataLookupStr(&mdok, in->header, "ZCMPTYPE");
+    psFitsCompressionType compType = psFitsCompressionTypeFromString(compTypeStr);
+    if (compType == PS_FITS_COMPRESS_NONE) {
+        return;
+    }
     // Get current BITPIX, BSCALE, BZERO, EXTNAME
     // Probably not necessary to look the numerical values up in this
@@ -609,8 +713,7 @@
 
 #ifdef STREAKS_COMPRESS_OUTPUT
-    // Paul says that I should be able to leave this blank
-    bitpix = 0;
-    setFitsOptions(out, bitpix, bscale, bzero);
-    setFitsOptions(rec, bitpix, bscale, bzero);
+    // printf("%d %f %f\n", bitpix, bscale, bzero);
+    setFitsOptions(out, bitpix, bscale, bzero, compType, tiles);
+    setFitsOptions(rec, bitpix, bscale, bzero, compType, tiles);
 #endif
 }
@@ -789,19 +892,27 @@
 
 bool
-replicate(sFile *sfile, void *xattr)
-{
-    if (!sfile->inNebulous) {
+replicate(sFile *outFile, sFile *inFile)
+{
+    if (!outFile->inNebulous) {
         return true;
     }
     nebServer *server = getNebServer(NULL);
 
-    // for now just set "user.copies" to 2
-    if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    char *user_copies = nebGetXattr(server, inFile->name, "user.copies");
+    bool free_user_copies = true;
+    if (user_copies == NULL) {
+        user_copies = "2";
+        free_user_copies = false;
+    }
+    if (!nebSetXattr(server, outFile->name, "user.copies", user_copies,  NEB_REPLACE)) {
+        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", outFile->name, nebErr(server));
         return false;
     }
-    if (!nebReplicate(server, sfile->name, NULL, NULL)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    if (!nebReplicate(server, outFile->name, "any", NULL)) {
+        psError(PM_ERR_UNKNOWN, true, "nebReplicate failed for %s\n%s", outFile->name, nebErr(server));
         return false;
+    }
+    if (free_user_copies) {
+        nebFree(user_copies);
     }
     return true;
@@ -816,35 +927,36 @@
     bool status = false;
 
-    // XXX: TODO: need a nebGetXatrr function, but there isn't one
-    // another option would be to take the number of copies to be
-    // created as an option. That way the system could decide
-    // whether to replicate anything other than raw Image files
-    void *xattr = NULL;
-
-    if (!replicate(sfiles->outImage, xattr)) {
+    if (!replicate(sfiles->outImage, sfiles->inImage)) {
         psError(PM_ERR_SYS, false, "failed to replicate outImage.");
         return false;
     }
 
-#ifdef notyet
-    // XXX: don't replicate mask and weight images until we can look up
-    // the input's xattr. There may be a perl program that can getXattr
     if (sfiles->outMask) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outMask, xattr)) {
+        if (!replicate(sfiles->outMask, sfiles->inMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-    if (sfiles->outWeight) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outWeight, xattr)) {
+    if (sfiles->outChMask) {
+        if (!replicate(sfiles->outChMask, sfiles->inChMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-#endif
-
-//      replicate the recovery images (if in nebulous)
+    if (sfiles->outWeight) {
+        if (!replicate(sfiles->outWeight, sfiles->inWeight)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+
+    if (sfiles->outSources) {
+        if (!replicate(sfiles->outSources, sfiles->inSources)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outSources.");
+            return false;
+        }
+    }
+
+//      XXX: replicate the recovery images (if in nebulous)
 //      perhaps whether we do that or not should be configurable.
 //      Sounds like we need a recipe
@@ -903,4 +1015,18 @@
     }
 
+    if (sfiles->outChMask) {
+        if (!swapOutputToInput(sfiles->inChMask, sfiles->outChMask)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for chip mask.");
+            return false;
+        }
+    }
+
+    if (sfiles->outSources) {
+        if (!swapOutputToInput(sfiles->inSources, sfiles->outSources)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for sources.");
+            return false;
+        }
+    }
+
     if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
         psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
@@ -908,4 +1034,5 @@
     }
 
+
     return true;
 }
@@ -914,6 +1041,4 @@
 deleteFile(sFile *sfile)
 {
-#if 0
-    // XXX API for nebDelete has changed; need to fix this later
     if (sfile->inNebulous) {
         nebServer *server = getNebServer(NULL);
@@ -930,5 +1055,4 @@
         }
     }
-#endif
     return true;
 }
@@ -938,20 +1062,21 @@
 {
     if (sfiles->outMask) {
-        if (!deleteFile(sfiles->outMask)) {
-            psError(PM_ERR_SYS, false, "failed to delete Mask.");
-            return false;
-        }
+        deleteFile(sfiles->outMask);
+    }
+
+    if (sfiles->outChMask) {
+        deleteFile(sfiles->outChMask);
     }
 
     if (sfiles->outWeight) {
-        if (!deleteFile(sfiles->outWeight)) {
-            psError(PM_ERR_SYS, false, "failed to delete Weight.");
-            return false;
-        }
-    }
-
-    if (!deleteFile(sfiles->outImage)) {
-        psError(PM_ERR_SYS, false, "failed to delete Image.");
-        return false;
+        deleteFile(sfiles->outWeight);
+    }
+
+    if (sfiles->outImage) {
+        deleteFile(sfiles->outImage);
+    }
+
+    if (sfiles->outSources) {
+        deleteFile(sfiles->outSources);
     }
 
@@ -1018,11 +1143,11 @@
         }
         if (printCounts) {
-            printf("time to NAN mask pixels: %f\n", psTimerClear("NAN_MASKED"));
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "time to NAN mask pixels: %f\n", psTimerClear("NAN_MASKED"));
             int totalPixels = image->numRows * image->numCols;
-            printf("pixels:        %10ld\n", totalPixels);
-            printf("masked pixels: %10ld %4.2f%%\n", maskedPixels, 100. * maskedPixels / totalPixels);
-            printf("nand pixels:   %10ld %4.2f%%\n", nandPixels, 100. * nandPixels / totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "pixels:        %10ld\n", totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "masked pixels: %10ld %4.2f%%\n", maskedPixels, 100. * maskedPixels / totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "nand pixels:   %10ld %4.2f%%\n", nandPixels, 100. * nandPixels / totalPixels);
             if (weight) {
-                printf("nand weights:  %10ld %4.2f%%\n", nandWeights, 100. * nandWeights / totalPixels);
+                psLogMsg(sfiles->program_name, PS_LOG_INFO, "nand weights:  %10ld %4.2f%%\n", nandWeights, 100. * nandWeights / totalPixels);
             }
         }
@@ -1050,5 +1175,5 @@
 strkGetMaskValues(streakFiles *sfiles, psU32 *maskStreak, psU32 *maskMask)
 {
-    if (sfiles->inMask->header) {
+    if (sfiles->inMask && sfiles->inMask->header) {
         if (!pmConfigMaskReadHeader(sfiles->config, sfiles->inMask->header)) {
             streaksExit("failed to read mask values from file", PS_EXIT_CONFIG_ERROR);
@@ -1080,2 +1205,59 @@
     *maskMask = ~convPoor;
 }
+
+static void
+doCopyExtraExtensions(streakFiles *sf, sFile *in, sFile *out)
+{
+    for (int extnum = sf->extnum; extnum < in->nHDU; extnum++) {
+        moveExt(in, extnum);
+        readImage(in, extnum, sf->stage, false);
+
+        if (SFILE_IS_IMAGE(in)) {
+            // Turn off compression (code adapted from pmFPAWrite)
+#ifdef SAVE_AND_RESTORE_COMPRESSION
+            int bitpix = out->fits->options ? out->fits->options->bitpix : 0; // Desired bits per pixel
+            psFitsCompression *compress = psFitsCompressionGet(out->fits); // Current compression options
+#endif
+            if (!psFitsSetCompression(out->fits, PS_FITS_COMPRESS_NONE, NULL, 0, 0, 0)) {
+                psError(PM_ERR_UNKNOWN, false, "failed to turn off compression for extension %d\n", extnum);
+                streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+            }
+#ifdef SAVE_AND_RESTORE_COMPRESSION
+            if (out->fits->options) {
+                out->fits->options->bitpix = 0;
+            }
+            if (!psFitsWriteImage(out->fits, in->header, in->image, 0, in->extname)) {
+                psError(PM_ERR_UNKNOWN, false, "failed to write image for extension %d\n", extnum);
+                streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+            }
+            if (out->fits->options) {
+                out->fits->options->bitpix = bitpix;
+            }
+            if (!psFitsCompressionApply(out->fits, compress)) {
+                psError(PM_ERR_UNKNOWN, false, "failed to rest compression image for extension %d\n", extnum);
+                streaksExit("", PS_EXIT_UNKNOWN_ERROR);
+            }
+#endif
+        } else {
+            copyTable(out, in, extnum);
+        }
+    }
+}
+
+void
+copyExtraExtensions(streakFiles *sf)
+{
+    // XXX: Bad assumption, the begining of the mask and weight files have the same structure as the image
+    // So we are currently at the image portion
+    sf->extnum = sf->inImage->nHDU;
+
+    if (sf->inWeight && sf->outWeight) {
+        doCopyExtraExtensions(sf, sf->inWeight, sf->outWeight);
+    }
+    if (sf->inMask && sf->outMask) {
+        doCopyExtraExtensions(sf, sf->inMask, sf->outMask);
+    }
+    if (sf->inChMask && sf->outChMask) {
+        doCopyExtraExtensions(sf, sf->inChMask, sf->outChMask);
+    }
+}