Changeset 24557 for branches/eam_branches/20090522/magic/remove/src/streaksio.c

Timestamp:

Jun 25, 2009, 2:00:56 PM (17 years ago)

Author:

eugene

Message:

merging changes from head

Location:

branches/eam_branches/20090522

Files:

• . (modified) (1 prop)
• magic/remove/src (modified) (1 prop)
• magic/remove/src/streaksio.c (modified) (20 diffs)

branches/eam_branches/20090522/magic/remove/src

@@ -3 +3 @@
 streakscompare
 streaksrelease
+makefile

@@ -3 +3 @@
 streakscompare
 streaksrelease
+makefile

branches/eam_branches/20090522/magic/remove/src/streaksio.c

@@ -19 +19 @@
     psMemSetDeallocator(sf, (psFreeFunc) streakFilesFree);
     memset(sf, 0, sizeof(*sf));
+
+    if (remove) {
+        // remember pointer so that streaksExit can delete temps
+        setStreakFiles(sf);
+    }
 
     sf->config = config;
@@ -160 +165 @@
 }
 
+// figure out if a nebulous instance is a non-destreaked file
+static bool
+nebFileIsDestreaked(sFile *sfile)
+{
+    if (!sfile->resolved_name) {
+        psError(PS_ERR_PROGRAMMING, true, "resolved name is null");
+        return false;
+    }
+    psFits *fits = psFitsOpen(sfile->resolved_name, "r");
+    if (!fits) {
+        psError(PS_ERR_IO, true, "failed open %s", sfile->name);
+        // can't tell if it is a destreaked file
+        return false;
+    }
+    psMetadata *header = psFitsReadHeader(NULL, fits);
+    if (!header) {
+        psError(PS_ERR_IO, true, "failed to read header for: %s", sfile->name);
+        return false;
+    }
+    bool mdok;
+    bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK");
+    if (mdok && isDestreaked) {
+        return true;
+    } else {
+        psError(PS_ERR_IO, false, "output file already exists and may not be de-streaked: %s", sfile->name);
+        return false;
+    }
+}
+
 static psString
 resolveFilename(pmConfig *config, sFile *sfile, bool create)
@@ -171 +205 @@
             // delete the existing file, since there may be more than one
             // instance. It will get created below in pmConfigConvertFilename
-            if (nebFind(server, sfile->name)) {
+            if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) {
+                if (!nebFileIsDestreaked(sfile)) {
+                    psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name);
+                    return NULL;
+                }
+                nebFree(sfile->resolved_name);
+                sfile->resolved_name = NULL;
                 nebDelete(server, sfile->name);
             }
@@ -193 +233 @@
     // all of the keywords in the raw image files written to the output destreaked files
 
-    if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
+    if (!outputFilename && !CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
         // stage is warp or diff AND fileSelect eq "INPUT"
         // get data from pmFPAfile.
@@ -250 +290 @@
     }
 
-    // if outputFilename is not null name it contains the "directory"
+    // if outputFilename is not null name it contains the "directory" (perhaps with a prefix like SR_)
     // and outputFilename is the basename name of the file (or nebulous key)
     // and the file is to be opened for writing
@@ -334 +374 @@
 
 void
+addDestreakKeyword(psMetadata *header)
+{
+    psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE,
+        "Have streaks been removed from image?", true);
+}
+
+void
+addRecoveryKeyword(psMetadata *header)
+{
+    psMetadataAddBool(header, PS_LIST_TAIL, "PSRECOVR", PS_META_REPLACE,
+        "Does this image contain excised streak pixels?", true);
+}
+
+void
 copyPHU(streakFiles *sfiles, bool remove)
 {
@@ -344 +398 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-
-    // TODO: add keyword indicating that streaks have been removed
+    psMetadata *recHeader = NULL;
+    if (remove && sfiles->recImage) {
+       recHeader = psMetadataCopy(NULL, imageHeader);
+       addRecoveryKeyword(recHeader);
+    }
+
+    // add keyword indicating that streaks have been removed
+    addDestreakKeyword(imageHeader);
+
     if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
         psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -351 +412 @@
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
-    // TODO: add keyword indicating that this is the recovery image
-    if (remove && sfiles->recImage && !psFitsWriteBlank(sfiles->recImage->fits, imageHeader, NULL)) {
+    if (recHeader && !psFitsWriteBlank(sfiles->recImage->fits, recHeader, NULL)) {
         psError(PS_ERR_IO, false, "failed to write primary header to %s",
             sfiles->recImage->resolved_name);
         streaksExit("", PS_EXIT_DATA_ERROR);
     }
+    psFree(recHeader);
+    recHeader = NULL;
     psFree(imageHeader);
 
@@ -367 +429 @@
             streaksExit("", 1);
         }
-        // TODO: add keyword indicating that streaks have been removed
+        if (remove && sfiles->recMask) {
+            recHeader = psMetadataCopy(NULL, maskHeader);
+            // add keyword indicating that this is the recovery image
+           addRecoveryKeyword(recHeader);
+        }
+        // add keyword indicating that streaks have been removed
+        addDestreakKeyword(maskHeader);
         if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -373 +441 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        // TODO: add keyword indicating that this is the recovery image
-        if (remove && sfiles->recMask && !psFitsWriteBlank(sfiles->recMask->fits, maskHeader, NULL)) {
+        if (recHeader && !psFitsWriteBlank(sfiles->recMask->fits, recHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
                 sfiles->recMask->resolved_name);
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
+        psFree(recHeader);
+        recHeader = NULL;
         psFree(maskHeader);
     }
@@ -389 +458 @@
             streaksExit("", 1);
         }
-        // TODO: add keyword indicating that streaks have been removed
+        if (remove && sfiles->recWeight) {
+            recHeader = psMetadataCopy(NULL, weightHeader);
+            // add keyword indicating that this is a recovery image
+           addRecoveryKeyword(recHeader);
+        }
+
+        // add keyword indicating that streaks have been removed
+        addDestreakKeyword(weightHeader);
         if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
@@ -395 +471 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        // TODO: add keyword indicating that this is a recovery image
-        if (remove && sfiles->recWeight && !psFitsWriteBlank(sfiles->recWeight->fits, weightHeader, NULL)) {
+        if (recHeader && !psFitsWriteBlank(sfiles->recWeight->fits, recHeader, NULL)) {
             psError(PS_ERR_IO, false, "failed to write primary header to %s",
                 sfiles->recWeight->resolved_name);
@@ -402 +477 @@
         }
         psFree(weightHeader);
+        psFree(recHeader);
     }
 }
@@ -566 +642 @@
 
 static void
-setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero)
+setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero, psFitsCompressionType compType,
+    psVector *tiles)
 {
     if (!sfile) {
@@ -579 +656 @@
     sfile->fits->options->bscale = bscale;
     sfile->fits->options->bzero = bzero;
-}
-
-void
-copyFitsOptions(sFile *out, sFile *rec, sFile *in)
-{
+
+    psFitsSetCompression(sfile->fits, compType, tiles, 8, 0, 0);
+}
+
+void
+copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles)
+{
+    bool mdok;
+    psString compTypeStr = psMetadataLookupStr(&mdok, in->header, "ZCMPTYPE");
+    psFitsCompressionType compType = psFitsCompressionTypeFromString(compTypeStr);
+    if (compType == PS_FITS_COMPRESS_NONE) {
+        return;
+    }
     // Get current BITPIX, BSCALE, BZERO, EXTNAME
     // Probably not necessary to look the numerical values up in this
@@ -610 +695 @@
 
 #ifdef STREAKS_COMPRESS_OUTPUT
-    // Paul says that I should be able to leave this blank
-    bitpix = 0;
-    setFitsOptions(out, bitpix, bscale, bzero);
-    setFitsOptions(rec, bitpix, bscale, bzero);
+    // printf("%d %f %f\n", bitpix, bscale, bzero);
+    setFitsOptions(out, bitpix, bscale, bzero, compType, tiles);
+    setFitsOptions(rec, bitpix, bscale, bzero, compType, tiles);
 #endif
 }
@@ -790 +874 @@
 
 bool
-replicate(sFile *sfile, void *xattr)
-{
-    if (!sfile->inNebulous) {
+replicate(sFile *outFile, sFile *inFile)
+{
+    if (!outFile->inNebulous) {
         return true;
     }
     nebServer *server = getNebServer(NULL);
 
-    // for now just set "user.copies" to 2
-    if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    char *user_copies = nebGetXattr(server, inFile->name, "user.copies");
+    bool free_user_copies = true;
+    if (user_copies == NULL) {
+        user_copies = "2";
+        free_user_copies = false;
+    }
+    if (!nebSetXattr(server, outFile->name, "user.copies", user_copies,  NEB_REPLACE)) {
+        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", outFile->name, nebErr(server));
         return false;
     }
-    if (!nebReplicate(server, sfile->name, NULL, NULL)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    if (!nebReplicate(server, outFile->name, "any", NULL)) {
+        psError(PM_ERR_UNKNOWN, true, "nebReplicate failed for %s\n%s", outFile->name, nebErr(server));
         return false;
+    }
+    if (free_user_copies) {
+        nebFree(user_copies);
     }
     return true;
@@ -817 +909 @@
     bool status = false;
 
-    // XXX: TODO: need a nebGetXatrr function, but there isn't one
-    // another option would be to take the number of copies to be
-    // created as an option. That way the system could decide
-    // whether to replicate anything other than raw Image files
-    void *xattr = NULL;
-
-    if (!replicate(sfiles->outImage, xattr)) {
+    if (!replicate(sfiles->outImage, sfiles->inImage)) {
         psError(PM_ERR_SYS, false, "failed to replicate outImage.");
         return false;
     }
 
-#ifdef notyet
-    // XXX: don't replicate mask and weight images until we can look up
-    // the input's xattr. There may be a perl program that can getXattr
     if (sfiles->outMask) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outMask, xattr)) {
+        if (!replicate(sfiles->outMask, sfiles->inMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-    if (sfiles->outWeight) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outWeight, xattr)) {
+    if (sfiles->outChMask) {
+        if (!replicate(sfiles->outChMask, sfiles->inChMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-#endif
-
-//      replicate the recovery images (if in nebulous)
+    if (sfiles->outWeight) {
+        if (!replicate(sfiles->outWeight, sfiles->inWeight)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+
+//      XXX: replicate the recovery images (if in nebulous)
 //      perhaps whether we do that or not should be configurable.
 //      Sounds like we need a recipe
@@ -904 +990 @@
     }
 
+    if (sfiles->outChMask) {
+        if (!swapOutputToInput(sfiles->inChMask, sfiles->outChMask)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for chip mask.");
+            return false;
+        }
+    }
+
     if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
         psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
@@ -936 +1029 @@
 {
     if (sfiles->outMask) {
-        if (!deleteFile(sfiles->outMask)) {
-            psError(PM_ERR_SYS, false, "failed to delete Mask.");
-            return false;
-        }
+        deleteFile(sfiles->outMask);
+    }
+
+    if (sfiles->outChMask) {
+        deleteFile(sfiles->outChMask);
     }
 
     if (sfiles->outWeight) {
-        if (!deleteFile(sfiles->outWeight)) {
-            psError(PM_ERR_SYS, false, "failed to delete Weight.");
-            return false;
-        }
-    }
-
-    if (!deleteFile(sfiles->outImage)) {
-        psError(PM_ERR_SYS, false, "failed to delete Image.");
-        return false;
+        deleteFile(sfiles->outWeight);
+    }
+
+    if (sfiles->outImage) {
+        deleteFile(sfiles->outImage);
     }
 

@@ -3 +3 @@
 streakscompare
 streaksrelease
+makefile