Index: /trunk/magic/remove/src/streaksremove.c =================================================================== --- /trunk/magic/remove/src/streaksremove.c (revision 20359) +++ /trunk/magic/remove/src/streaksremove.c (revision 20360) @@ -1,8 +1,11 @@ #include "streaks.h" #include "streaksextern.h" +#include "libgen.h" extern bool sFileLock(sFile * sfile); extern bool sFileUnlock(sFile * sfile); extern bool sFileReplace(sFile *dest, sFile *src); + +static nebServer *ourNebServer = NULL; // for clarity in this program we don't propagate errors up the stack @@ -13,6 +16,5 @@ } -static void -sFileFree(sFile *sfile) +static void sFileFree(sFile *sfile) { psFree(sfile->resolved_name); @@ -21,8 +23,66 @@ } +// getNebServer() +// +// it gets created the first time a nebulous file is accessed. +// if config is passed in we are to create it and exit if not able. +// if config is null we return NULL indicating that there are no nebulous files in use +// The purpose of this is to avoid passing extra arguments between all of the functions. +// but it is probably a bad idea. +// +static nebServer *getNebServer(pmConfig *config) +{ + if (ourNebServer) { + return ourNebServer; + } + + if (!config) { + return NULL; + } + + psString serverURI = getenv("NEB_SERVER"); + + // XXX: Note: all of the flags to psError should be true, but I don't think nebclient + // uses psError + if (!serverURI) { + bool status; + serverURI = psMetadataLookupStr(&status, config->site, "NEB_SERVER"); + if (!status) { + psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER."); + streaksremoveExit("", PS_EXIT_CONFIG_ERROR); + } + } + + if (!serverURI) { + psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI."); + + streaksremoveExit("", PS_EXIT_CONFIG_ERROR); + } + + ourNebServer = nebServerAlloc(serverURI); + if (!ourNebServer) { + psError(PM_ERR_SYS, true, "failed to create a nebServer object from %s.", serverURI); + streaksremoveExit("", PS_EXIT_CONFIG_ERROR); + } + + return ourNebServer; +} + psString resolveFilename(pmConfig *config, sFile *sfile, bool create) { sfile->inNebulous = IN_NEBULOUS(sfile->name); + + if (sfile->inNebulous) { + // make sure we have our neb server connection + nebServer *server = getNebServer(config); + if (create) { + // delete the existing file, since there may be more than one + // instance. It will get created below in pmConfigConvertFilename + if (nebFind(server, sfile->name)) { + nebDelete(server, sfile->name); + } + } + } return pmConfigConvertFilename(sfile->name, config, create, create); @@ -37,6 +97,6 @@ // We use psFits directly to read the image file unless the stage is warp - // or diff. In that case we use the pmFPAfile functions to read the image file - // to make managing the astromety easy + // or diff. In those cases we use the pmFPAfile functions to read the image file + // to make managing the astrometry easy if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) { @@ -67,5 +127,4 @@ } - sfile->resolved_name = psStringCopy(sfile->pmfile->filename); pmFPAview *view = pmFPAviewAlloc(0); if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { @@ -74,4 +133,5 @@ } psFree(view); + sfile->resolved_name = psStringCopy(sfile->pmfile->filename); // copy header from fpu @@ -80,4 +140,6 @@ return sfile; } + + // using psFits for i/o sfile->name = psMetadataLookupStr(&status, config->arguments, fileSelect); @@ -211,26 +273,32 @@ sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true); sf->nHDU = sf->inImage->nHDU; - sf->outImage = sFileOpen(config, stage, "OUTPUT", ".fits", true); - sf->recImage = sFileOpen(config, stage, "RECOVERY", ".fits", true); + + // don't need to free this see basename(3) + char *inputBasename = basename(sf->inImage->name); + sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, true); sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false); if (sf->inMask) { - sf->outMask = sFileOpen(config, stage, "OUTPUT", ".mk.fits", true); - sf->recMask = sFileOpen(config, stage, "RECOVERY", "wt.fits", true); + inputBasename = basename(sf->inMask->name); + sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, true); } sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false); if (sf->inWeight) { - sf->outWeight = sFileOpen(config, stage, "OUTPUT", ".wt.fits", true); - sf->recWeight = sFileOpen(config, stage, "RECOVERY", ".wt.fits", true); - } - - // load astrometry if supplied + inputBasename = basename(sf->inWeight->name); + sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, true); + } + + // load astrometry file if (USE_SUPPLIED_ASTROM(sf->stage)) { sf->inAstrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM"); } else { - sf->inAstrom = sf->inImage->pmfile; + // otherwise get astrometry from pmfile if (!sf->inImage->pmfile) { streaksremoveExit("unexpected null pmFPAfile", PS_EXIT_CONFIG_ERROR); } + sf->inAstrom = sf->inImage->pmfile; } setupAstromFromFPA(sf); @@ -316,4 +384,16 @@ } +psString checkdir(char *path) +{ + // insure that path ends in / + int len = strlen(path); + psString dir = NULL; + if (strrchr(path, '/') != (path + len - 1)) { + psStringAppend(&dir, "%s/", path); + } else { + dir = psStringCopy(path); + } + return dir; +} ippStage @@ -378,6 +458,11 @@ } + + // If input image is in nebulous we require mask, weight, and tmproot to all be + // nebulous paths + bool nebulousImage = false; if ((argnum = psArgumentGet(argc, argv, "-image"))) { // for raw and chip level images we use psFits for I/O and ... + nebulousImage = IN_NEBULOUS(argv[argnum+1]); if (CHIP_LEVEL_INPUT(stage)) { psArgumentRemove(argnum, &argc, argv); @@ -406,4 +491,10 @@ if ((argnum = psArgumentGet(argc, argv, "-mask"))) { psArgumentRemove(argnum, &argc, argv); + bool nebulousMask = IN_NEBULOUS(argv[argnum]); + if (nebulousMask != nebulousImage) { + psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n", + nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular"); + return NULL; + } psMetadataAddStr(config->arguments, PS_LIST_TAIL, "MASK", 0, "name of input mask image", argv[argnum]); @@ -412,4 +503,10 @@ if ((argnum = psArgumentGet(argc, argv, "-weight"))) { psArgumentRemove(argnum, &argc, argv); + bool nebulousWeight = IN_NEBULOUS(argv[argnum]); + if (nebulousWeight != nebulousImage) { + psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n", + nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular"); + return NULL; + } psMetadataAddStr(config->arguments, PS_LIST_TAIL, "WEIGHT", 0, "name of input weight image", argv[argnum]); @@ -418,15 +515,23 @@ if ((argnum = psArgumentGet(argc, argv, "-tmproot"))) { psArgumentRemove(argnum, &argc, argv); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "root name of temporary files", - argv[argnum]); - psArgumentRemove(argnum, &argc, argv); - } else { - psError(PS_ERR_UNKNOWN, true, "-outroot is required\n"); + bool nebulousTmp = IN_NEBULOUS(argv[argnum]); + if (nebulousTmp != nebulousImage) { + psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path\n", + nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular"); + return NULL; + } + psString dir = checkdir(argv[argnum]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files", + dir); + psArgumentRemove(argnum, &argc, argv); + } else { + psError(PS_ERR_UNKNOWN, true, "-tmproot is required\n"); return NULL; } if ((argnum = psArgumentGet(argc, argv, "-recovery"))) { psArgumentRemove(argnum, &argc, argv); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "Name of recovery root", - argv[argnum]); + psString dir = checkdir(argv[argnum]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files", + dir); psArgumentRemove(argnum, &argc, argv); } else if (stage == IPP_STAGE_RAW) { @@ -434,5 +539,7 @@ return NULL; } + bool gotReplace = false; if ((argnum = psArgumentGet(argc, argv, "-replace"))) { + gotReplace = true; psArgumentRemove(argnum, &argc, argv); psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files", @@ -440,7 +547,21 @@ } if ((argnum = psArgumentGet(argc, argv, "-remove"))) { - psArgumentRemove(argnum, &argc, argv); - psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove input files", + if (!gotReplace) { + psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n"); + return NULL; + } + psArgumentRemove(argnum, &argc, argv); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files", true); + } + + if (argc != 1) { + psString unexpectedArguments = NULL; + for (int i=1; inumRows with in->image->numRows rec->image = psImageRecycle(rec->image, in->image->numCols, in->image->numRows, in->image->type.type); if (!rec->image) { @@ -879,9 +999,11 @@ static bool -replicate(nebServer *server, sFile *sfile, void *xattr) +replicate(sFile *sfile, void *xattr) { if (!sfile->inNebulous) { return true; } + nebServer *server = getNebServer(NULL); + // for now just set "user.copies" to 2 if (!nebSetXattr(server, sfile->name, "user.copies", "2", NEB_REPLACE)) { @@ -903,31 +1025,12 @@ { bool status = false; - psString neb_server = getenv("NEB_SERVER"); - - // XXX: Note: all of the flags to psError should be true, but I don't think nebclient - // uses psError - if (!neb_server) { - neb_server = psMetadataLookupStr(&status, sfiles->config->site, "NEB_SERVER"); - if (!status) { - psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER."); - return false; - } - } - - if (!neb_server) { - psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI."); - return false; - } - - sfiles->nebServer = nebServerAlloc(neb_server); - if (!sfiles->nebServer) { - psError(PM_ERR_SYS, true, "failed to create a nebServer object."); - return false; - } // XXX: TODO: need a nebGetXatrr function, but there isn't one + // another option would be to take the number of copies to be + // created as an option. That way the system could decide + // whether to replicate anything other than raw Image files void *xattr = NULL; - if (!replicate(sfiles->nebServer, sfiles->outImage, xattr)) { + if (!replicate(sfiles->outImage, xattr)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; @@ -935,9 +1038,9 @@ #ifdef notyet - // don't replicate mask and weight images until we can look up + // XXX: don't replicate mask and weight images until we can look up // the input's xattr. There may be a perl program that can getXattr if (sfiles->outMask) { // get xattr from input to see if we need to replicate - if (!replicate(sfiles->nebServer, sfiles->outMask, xattr)) { + if (!replicate(sfiles->outMask, xattr)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; @@ -946,5 +1049,5 @@ if (sfiles->outWeight) { // get xattr from input to see if we need to replicate - if (!replicate(sfiles->nebServer, sfiles->outWeight, xattr)) { + if (!replicate(sfiles->outWeight, xattr)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; @@ -959,32 +1062,60 @@ return true; } + +static bool +swapOutputToInput(sFile *in, sFile *out) +{ + + if (in->inNebulous) { + nebServer *server = getNebServer(NULL); + + if (!nebSwap(server, in->name, out->name)) { + psError(PM_ERR_SYS, true, "failed to swap files for: %s.", + in->name, nebErr(server)); + return false; + } + } else { + psString command = NULL; + + // regular files. use mv creating a backup + // NOTE: -b is a gnu option and may not always be available + psStringAppend(&command, "mv -b --suffix=.bak %s %s", out->resolved_name, in->resolved_name); + int status = system(command); + if (status != 0) { + psError(PM_ERR_SYS, true, "%s failed with status %d.", + command, status); + psFree(command); + return false; + } + psFree(command); + } + + return true; +} static bool swapOutputsToInputs(streakFiles *sfiles) { - bool status = false; - - if (!sfiles->nebServer) { - // no output files are in nebulous - return true; - } - - // XXX: we should check for consistency before here - // We need to insure that the database is consistent - // If input is in nebulous, output must be - // if input is not in nebulous output must not be Catch in parseArguments - // if not in nebulous create a backup of input before overwriting - - if (sfiles->inImage->inNebulous && sfiles->outImage->inNebulous) { - if (!nebSwap(sfiles->nebServer, sfiles->outImage->name, sfiles->inImage->name)) { - psError(PM_ERR_SYS, false, "failed to swap instances for Image\n%s.", - nebErr(sfiles->nebServer)); + if (sfiles->inMask) { + if (!swapOutputToInput(sfiles->inMask, sfiles->outMask)) { + psError(PM_ERR_SYS, false, "failed to swap instances for Mask."); return false; } } - // TODO: handle weight and mask images + if (sfiles->inWeight) { + if (!swapOutputToInput(sfiles->inWeight, sfiles->outWeight)) { + psError(PM_ERR_SYS, false, "failed to swap instances for Weight."); + return false; + } + } + + if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) { + psError(PM_ERR_SYS, false, "failed to swap instances for Image."); + return false; + } return true; } + static bool deleteTemps(streakFiles *sfiles) @@ -992,19 +1123,16 @@ bool status = false; - if (!sfiles->nebServer) { - // no output files are in nebulous + nebServer *server = getNebServer(NULL); + + if (!server) { + // TODO: handle this + // + // no files are in nebulous return true; } - - // XXX: we should check for consistency before here - // We need to insure that the database is consistent - // If input is in nebulous, output must be - // if input is not in nebulous output must not be Catch in parseArguments - // if not in nebulous create a backup of input before overwriting - if (sfiles->outImage->inNebulous) { - if (!nebDelete(sfiles->nebServer, sfiles->outImage->name)) { + if (!nebDelete(server, sfiles->outImage->name)) { psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfiles->outImage->name, - nebErr(sfiles->nebServer)); + nebErr(getNebServer(NULL))); return false; } @@ -1031,4 +1159,9 @@ psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS"); Streaks *streaks = readStreaksFile(streaksFileName); + + if (!streaks) { + psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName); + exit (PS_EXIT_PROG_ERROR); + } streakFiles *sfiles = openFiles(config); @@ -1085,4 +1218,5 @@ #endif + // XXX: This loop following will SEGV for raw video Cells becuase the image is null for (i = 0; i < psArrayLength (pixels); ++i) { pixelPos = psArrayGet (pixels, i); @@ -1127,22 +1261,27 @@ // Note this is a database operation. No file I/O is performed if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to replicate output files"); + psError(PS_ERR_UNKNOWN, false, "failed to swap files"); + // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that // it has done and give a detailed report of what happened + psErrorStackPrint(stderr, ""); exit(PS_EXIT_UNKNOWN_ERROR); } - } - - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - // Returning error status is problematic. Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); + + if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { + // delete the temporary storage objects (which now points to the original image(s) + if (!deleteTemps(sfiles)) { + psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); + // XXX: Now what? At this point the output files have been swapped, so we can't + // repeat the operation. + + // Returning error status here is problematic. The inputs have been streak removed + // but they're still lying around + // Maybe just print an error message and + // let other system tools clean up + psErrorStackPrint(stderr, ""); + exit(PS_EXIT_UNKNOWN_ERROR); + } } }