Index: /trunk/magic/remove/src/Makefile.simple =================================================================== --- /trunk/magic/remove/src/Makefile.simple (revision 24555) +++ /trunk/magic/remove/src/Makefile.simple (revision 24556) @@ -28,4 +28,11 @@ streaksrelease.o +ISDESTREAKED_OBJECTS= \ + ${COMMON_OBJECTS} \ + isdestreaked.o + + +HEADERS= Line.h streaksastrom.h streaksextern.h streaksio.h streaksremove.h + STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1 OPTFLAGS= -g -O2 @@ -35,8 +42,10 @@ LDFLAGS=`psmodules-config --libs` -PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease +PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease isdestreaked +HEADERS=Line.h streaksastrom.h streaksextern.h streaksio.h streaksremove.h all: ${PROGRAMS} +${REMOVE_OBJECTS}: ${HEADERS} streaksremove: ${REMOVE_OBJECTS} @@ -47,4 +56,7 @@ streaksrelease: ${RELEASE_OBJECTS} +isdestreaked: ${ISDESTREAKED_OBJECTS} + + install: ${PROGRAMS} install -t $(PSCONFDIR)/$(PSCONFIG)/bin streaksremove @@ -52,4 +64,5 @@ install -t $(PSCONFDIR)/$(PSCONFIG)/bin streakscompare install -t $(PSCONFDIR)/$(PSCONFIG)/bin streaksrelease + install -t $(PSCONFDIR)/$(PSCONFIG)/bin isdestreaked clean: Index: /trunk/magic/remove/src/isdestreaked.c =================================================================== --- /trunk/magic/remove/src/isdestreaked.c (revision 24556) +++ /trunk/magic/remove/src/isdestreaked.c (revision 24556) @@ -0,0 +1,40 @@ +#include "streaksremove.h" + +int +main(int argc, char *argv[]) +{ + pmConfig *config = pmConfigRead(&argc, argv, NULL); + if (!config) { + psErrorStackPrint(stderr, "failed to parse arguments\n"); + exit(PS_EXIT_CONFIG_ERROR); + } + + char *fileName = argv[1]; + if (!fileName) { + fprintf(stderr, "Usage: %s filename\n", argv[0]); + exit(PS_EXIT_DATA_ERROR); + } + + psFits *fits = psFitsOpen(fileName, "r"); + if (!fits) { + psError(PS_ERR_UNKNOWN, false, "failed to open fits file: %s\n", fileName); + psErrorStackPrint(stderr, ""); + exit(PS_EXIT_DATA_ERROR); + return false; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psError(PS_ERR_IO, false, "failed to read header for: %s", fileName); + psErrorStackPrint(stderr, ""); + return false; + } + + bool mdok; + bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK"); + if (mdok && isDestreaked) { + exit(0); + } else { + exit(42); + } +} Index: /trunk/magic/remove/src/streaksio.c =================================================================== --- /trunk/magic/remove/src/streaksio.c (revision 24555) +++ /trunk/magic/remove/src/streaksio.c (revision 24556) @@ -19,4 +19,9 @@ psMemSetDeallocator(sf, (psFreeFunc) streakFilesFree); memset(sf, 0, sizeof(*sf)); + + if (remove) { + // remember pointer so that streaksExit can delete temps + setStreakFiles(sf); + } sf->config = config; @@ -160,4 +165,33 @@ } +// figure out if a nebulous instance is a non-destreaked file +static bool +nebFileIsDestreaked(sFile *sfile) +{ + if (!sfile->resolved_name) { + psError(PS_ERR_PROGRAMMING, true, "resolved name is null"); + return false; + } + psFits *fits = psFitsOpen(sfile->resolved_name, "r"); + if (!fits) { + psError(PS_ERR_IO, true, "failed open %s", sfile->name); + // can't tell if it is a destreaked file + return false; + } + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psError(PS_ERR_IO, true, "failed to read header for: %s", sfile->name); + return false; + } + bool mdok; + bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK"); + if (mdok && isDestreaked) { + return true; + } else { + psError(PS_ERR_IO, false, "output file already exists and may not be de-streaked: %s", sfile->name); + return false; + } +} + static psString resolveFilename(pmConfig *config, sFile *sfile, bool create) @@ -171,5 +205,11 @@ // delete the existing file, since there may be more than one // instance. It will get created below in pmConfigConvertFilename - if (nebFind(server, sfile->name)) { + if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) { + if (!nebFileIsDestreaked(sfile)) { + psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name); + return NULL; + } + nebFree(sfile->resolved_name); + sfile->resolved_name = NULL; nebDelete(server, sfile->name); } @@ -193,5 +233,5 @@ // all of the keywords in the raw image files written to the output destreaked files - if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) { + if (!outputFilename && !CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) { // stage is warp or diff AND fileSelect eq "INPUT" // get data from pmFPAfile. @@ -250,5 +290,5 @@ } - // if outputFilename is not null name it contains the "directory" + // if outputFilename is not null name it contains the "directory" (perhaps with a prefix like SR_) // and outputFilename is the basename name of the file (or nebulous key) // and the file is to be opened for writing @@ -834,19 +874,27 @@ bool -replicate(sFile *sfile, void *xattr) -{ - if (!sfile->inNebulous) { +replicate(sFile *outFile, sFile *inFile) +{ + if (!outFile->inNebulous) { return true; } nebServer *server = getNebServer(NULL); - // for now just set "user.copies" to 2 - if (!nebSetXattr(server, sfile->name, "user.copies", "2", NEB_REPLACE)) { - psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server)); + char *user_copies = nebGetXattr(server, inFile->name, "user.copies"); + bool free_user_copies = true; + if (user_copies == NULL) { + user_copies = "2"; + free_user_copies = false; + } + if (!nebSetXattr(server, outFile->name, "user.copies", user_copies, NEB_REPLACE)) { + psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", outFile->name, nebErr(server)); return false; } - if (!nebReplicate(server, sfile->name, NULL, NULL)) { - psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server)); + if (!nebReplicate(server, outFile->name, "any", NULL)) { + psError(PM_ERR_UNKNOWN, true, "nebReplicate failed for %s\n%s", outFile->name, nebErr(server)); return false; + } + if (free_user_copies) { + nebFree(user_copies); } return true; @@ -861,35 +909,29 @@ bool status = false; - // XXX: TODO: need a nebGetXatrr function, but there isn't one - // another option would be to take the number of copies to be - // created as an option. That way the system could decide - // whether to replicate anything other than raw Image files - void *xattr = NULL; - - if (!replicate(sfiles->outImage, xattr)) { + if (!replicate(sfiles->outImage, sfiles->inImage)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } -#ifdef notyet - // XXX: don't replicate mask and weight images until we can look up - // the input's xattr. There may be a perl program that can getXattr if (sfiles->outMask) { - // get xattr from input to see if we need to replicate - if (!replicate(sfiles->outMask, xattr)) { + if (!replicate(sfiles->outMask, sfiles->inMask)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } } - if (sfiles->outWeight) { - // get xattr from input to see if we need to replicate - if (!replicate(sfiles->outWeight, xattr)) { + if (sfiles->outChMask) { + if (!replicate(sfiles->outChMask, sfiles->inChMask)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } } -#endif - -// replicate the recovery images (if in nebulous) + if (sfiles->outWeight) { + if (!replicate(sfiles->outWeight, sfiles->inWeight)) { + psError(PM_ERR_SYS, false, "failed to replicate outImage."); + return false; + } + } + +// XXX: replicate the recovery images (if in nebulous) // perhaps whether we do that or not should be configurable. // Sounds like we need a recipe @@ -954,4 +996,5 @@ } } + if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) { psError(PM_ERR_SYS, false, "failed to swap instances for Image."); @@ -986,20 +1029,17 @@ { if (sfiles->outMask) { - if (!deleteFile(sfiles->outMask)) { - psError(PM_ERR_SYS, false, "failed to delete Mask."); - return false; - } + deleteFile(sfiles->outMask); + } + + if (sfiles->outChMask) { + deleteFile(sfiles->outChMask); } if (sfiles->outWeight) { - if (!deleteFile(sfiles->outWeight)) { - psError(PM_ERR_SYS, false, "failed to delete Weight."); - return false; - } - } - - if (!deleteFile(sfiles->outImage)) { - psError(PM_ERR_SYS, false, "failed to delete Image."); - return false; + deleteFile(sfiles->outWeight); + } + + if (sfiles->outImage) { + deleteFile(sfiles->outImage); } Index: /trunk/magic/remove/src/streaksio.h =================================================================== --- /trunk/magic/remove/src/streaksio.h (revision 24555) +++ /trunk/magic/remove/src/streaksio.h (revision 24556) @@ -20,5 +20,5 @@ void writeImage(sFile *sfile, psString extname, int extnum); void writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum); -bool replicate(sFile *sfile, void *xattr); +bool replicate(sFile *outFile, sFile *inFile); void readImageFrom_pmFile(streakFiles *sf); Index: /trunk/magic/remove/src/streaksremove.c =================================================================== --- /trunk/magic/remove/src/streaksremove.c (revision 24555) +++ /trunk/magic/remove/src/streaksremove.c (revision 24556) @@ -41,5 +41,5 @@ Streaks *streaks = readStreaksFile(streaksFileName); if (!streaks) { - psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName); + psError(PS_ERR_UNKNOWN, "failed to read streaks file: %s", streaksFileName); streaksExit("", PS_EXIT_PROG_ERROR); } @@ -209,4 +209,5 @@ if (!replicateOutputs(sfiles)) { psError(PS_ERR_UNKNOWN, false, "failed to replicate output files"); + deleteTemps(sfiles); psErrorStackPrint(stderr, ""); exit(PS_EXIT_UNKNOWN_ERROR); @@ -215,4 +216,6 @@ // NOTE: from here on we can't just quit if something goes wrong. // especially if we're working at the raw stage + // turn off automatic deletion of output files by streaksExit + setStreakFiles(NULL); if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) { @@ -220,36 +223,13 @@ // Note this is a nebulous database operation. No file I/O is performed if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to swap files"); - - // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that - // it has done and give a detailed report of what happened - - psErrorStackPrint(stderr, ""); + // XXX: Now what? + // It is up to the program that reverts failed destreak runs to insure that + // any input files that have been swapped are restored + + psErrorStackPrint(stderr, "failed to swap files"); // XXX: pick a specific error code for this failure exit(PS_EXIT_UNKNOWN_ERROR); } - -#ifdef notdef - // XXX: we've decided to not do the remove step here - // Instead we leave the backup images (the original images) in place and remove them - // later on - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - - // Returning error status here is problematic. The inputs have been streak removed - // but they're still lying around - // Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - } -#endif - } // all done. Clean up to look for memory leaks. @@ -446,8 +426,6 @@ usage(); } - psString dir = pathToDirectory(argv[argnum]); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files", - dir); - psFree(dir); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "path for (temporary if replae) output files", + argv[argnum]); psArgumentRemove(argnum, &argc, argv); } else { @@ -458,8 +436,6 @@ if ((argnum = psArgumentGet(argc, argv, "-recovery"))) { psArgumentRemove(argnum, &argc, argv); - psString dir = pathToDirectory(argv[argnum]); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files", - dir); - psFree(dir); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "path for recovery files", + argv[argnum]); psArgumentRemove(argnum, &argc, argv); } else if ((stage == IPP_STAGE_RAW) && gotReplace) { @@ -467,17 +443,4 @@ usage(); } - -#ifdef notdef - // This parameter is no longer supported - if ((argnum = psArgumentGet(argc, argv, "-remove"))) { - if (!gotReplace) { - psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n"); - usage(); - } - psArgumentRemove(argnum, &argc, argv); - psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files", - true); - } -#endif if (argc != 1) { Index: /trunk/magic/remove/src/streaksremove.h =================================================================== --- /trunk/magic/remove/src/streaksremove.h (revision 24555) +++ /trunk/magic/remove/src/streaksremove.h (revision 24556) @@ -88,4 +88,5 @@ extern ippStage parseStage(psString); extern psString pathToDirectory(char *path); +extern void setStreakFiles( streakFiles *); #define CHIP_LEVEL_INPUT(_stage) ((_stage == IPP_STAGE_RAW) || (_stage == IPP_STAGE_CHIP)) @@ -95,3 +96,4 @@ #define IN_NEBULOUS(_filename) (!strncasecmp(_filename, "neb://", strlen("neb://"))) + #endif // STREAKS_H Index: /trunk/magic/remove/src/streaksreplace.c =================================================================== --- /trunk/magic/remove/src/streaksreplace.c (revision 24555) +++ /trunk/magic/remove/src/streaksreplace.c (revision 24556) @@ -97,35 +97,4 @@ } -#ifdef NOTYET - if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) { - // swap the instances for the input and output - // Note this is a database operation. No file I/O is performed - if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to swap files"); - - // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that - // it has done and give a detailed report of what happened - - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - - // Returning error status here is problematic. The inputs have been streak removed - // but they're still lying around - // Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - } - } -#endif // REPLACE, REMOVE // nebServerFree(ourNebServer); psFree(config); Index: /trunk/magic/remove/src/streaksutil.c =================================================================== --- /trunk/magic/remove/src/streaksutil.c (revision 24555) +++ /trunk/magic/remove/src/streaksutil.c (revision 24556) @@ -33,8 +33,19 @@ } +streakFiles *ourStreakFiles = NULL; + +void +setStreakFiles(streakFiles *sfiles) +{ + ourStreakFiles = sfiles; +} + // to enhance clarity in these programs we don't propagate errors up the stack // we just bail out void streaksExit(psString str, int exitCode) { psErrorStackPrint(stderr, str); + if (ourStreakFiles) { + deleteTemps(ourStreakFiles); + } exit(exitCode); }