Index: branches/eam_branches/ipp-20101205/Nebulous/bin/neb-cull =================================================================== --- branches/eam_branches/ipp-20101205/Nebulous/bin/neb-cull (revision 30446) +++ branches/eam_branches/ipp-20101205/Nebulous/bin/neb-cull (revision 30447) @@ -17,5 +17,5 @@ my ($min_copies, $server, $volume, $one_only); - +my ($soft_volume); $server = $ENV{'NEB_SERVER'} unless $server; @@ -25,4 +25,5 @@ 'volume|v=s' => \$volume, 'one_only|o' => \$one_only, + 'soft_volume' => \$soft_volume, ) || pod2usage( 2 ); @@ -42,4 +43,10 @@ unless defined $neb; +if ($volume) { + unless (($soft_volume)) { + $volume = '~' . $volume; + } +} + $volume ||= "any"; $min_copies ||= 2; @@ -48,4 +55,8 @@ push @cull_args, $key; push @cull_args, $volume; + +# print "$key $volume\n"; +# print "@cull_args\n"; + if ($one_only) { @@ -95,4 +106,11 @@ Symbolic name of the volume to create the new instance on. + +Optional. + +=item * --soft_volume + +Treat the specified volume name as soft, and allow an instance to be culled +from another volume if the specified volume does not have an instance. Optional. Index: branches/eam_branches/ipp-20101205/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- branches/eam_branches/ipp-20101205/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30446) +++ branches/eam_branches/ipp-20101205/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30447) @@ -1048,4 +1048,29 @@ } +sub magic_filename +{ + my $file = shift; + my $prefix = shift; + my $dirname = dirname($file); + my $base = basename($file); + + return "$dirname/$prefix$base"; +} + +sub destreaked_filename +{ + my $self = shift; + my $file = shift; + + return magic_filename($file, "SR_"); +} +sub recovery_filename +{ + my $self = shift; + my $file = shift; + + return magic_filename($file, "REC_"); +} + # Return an EXTNAME From the EXTNAME.RULE table in the camera configuration sub extname_rule Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksVersion.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksVersion.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksVersion.c (revision 30447) @@ -58,5 +58,5 @@ psString source = streaksSource(); // Software source - psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", 0, NULL, source); + psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", PS_META_REPLACE, NULL, source); psStringPrepend(&version, "%s version: ", streaksProgram); Index: branches/eam_branches/ipp-20101205/magic/remove/src/streakscompare.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streakscompare.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streakscompare.c (revision 30447) @@ -18,6 +18,6 @@ ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE"); - sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true); - sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true); + sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true, false); + sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true, false); int ncomponents; Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.c (revision 30447) @@ -42,5 +42,5 @@ sf->class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID"); - sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true); + sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true, false); sf->nHDU = sf->inImage->nHDU; @@ -48,22 +48,22 @@ // The names of the temporary and recovery files are taken from the input char *inputBasename = basename(sf->inImage->name); - sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true); if (remove) { // XXX: the recovery file should be required if stage is raw and the replace flag is set // that is if the input raw image is to be destroyed - sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false); - } else { - sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true); - } - - sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false); + sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false); + } else { + sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true, false); + } + + sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false, false); if (sf->inMask && (sf->stage != IPP_STAGE_RAW)) { inputBasename = basename(sf->inMask->name); - sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true); if (remove) { - sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false); + sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false); } else { - sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true); + sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true, false); } } @@ -73,27 +73,27 @@ // If it doesn't exist, we didn't have a camera mask if (remove && sf->inMask && (stage == IPP_STAGE_CHIP)) { - sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false); + sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false, false); if (sf->inChMask) { inputBasename = basename(sf->inChMask->name); - sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true); - sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false); - } - } - - sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false); + sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true); + sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false); + } + } + + sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false, false); if (sf->inWeight) { inputBasename = basename(sf->inWeight->name); - sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true); if (remove) { - sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false); + sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false); } else { - sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true); + sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true, false); } } if (remove) { - sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false); + sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false, false); if (sf->inSources) { inputBasename = basename(sf->inSources->name); - sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true); + sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true); } } @@ -121,14 +121,4 @@ sf->transparentStreaks = psMetadataLookupF64(&status, config->arguments, "TRANSPARENT_STREAKS"); - sf->stats = psMetadataAlloc(); - psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS"); - if (statsFileName) { - sf->statsFile = fopen(statsFileName, "w"); - if (!sf->statsFile) { - psError(PS_ERR_IO, true, "failed to open stats file %s", statsFileName); - streaksExit("", PS_EXIT_CONFIG_ERROR); - } - } - return sf; } @@ -138,4 +128,5 @@ { freeImages(sf); + psFree(sf->stats); psFree(sf->diffedPixels); psFree(sf->tiles); @@ -221,5 +212,5 @@ static psString -resolveFilename(pmConfig *config, sFile *sfile, bool create) +resolveFilename(pmConfig *config, sFile *sfile, bool create, bool checkIfDestreaked) { sfile->inNebulous = IN_NEBULOUS(sfile->name); @@ -232,5 +223,5 @@ // instance. It will get created below in pmConfigConvertFilename if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) { - if (!nebFileIsDestreaked(sfile)) { + if (checkIfDestreaked && !nebFileIsDestreaked(sfile)) { psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name); return NULL; @@ -247,5 +238,5 @@ sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect, - psString outputFilename, bool required) + psString outputFilename, bool required, bool checkIfDestreaked) { bool status; @@ -322,6 +313,7 @@ // and the file is to be opened for writing if (outputFilename) { + sfile->write = true; psStringAppend(&sfile->name, "%s%s", name, outputFilename); - sfile->resolved_name = resolveFilename(config, sfile, true); + sfile->resolved_name = resolveFilename(config, sfile, true, checkIfDestreaked); if (!sfile->resolved_name) { psError(PS_ERR_IO, false, "Failed to resolve filename for %s", sfile->name); @@ -335,5 +327,5 @@ } else { sfile->name = psStringCopy(name); - sfile->resolved_name = resolveFilename(config, sfile, false); + sfile->resolved_name = resolveFilename(config, sfile, false, false); if (!sfile->resolved_name) { psError(PS_ERR_IO, false, "Failed to resolve name for %s", sfile->name); @@ -402,8 +394,8 @@ void -addDestreakKeyword(psMetadata *header) +addDestreakKeyword(psMetadata *header, bool value) { psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE, - "Have streaks been removed from image?", true); + "Have streaks been removed from image?", value); } @@ -416,5 +408,5 @@ void -copyPHU(streakFiles *sfiles, bool remove) +copyPHU(streakFiles *sfiles, bool remove, bool destreak) { psAssert(sfiles->stage == IPP_STAGE_RAW, "copyPHU should only be used for raw stage"); @@ -433,5 +425,5 @@ // add keyword indicating that streaks have been removed - addDestreakKeyword(imageHeader); + addDestreakKeyword(imageHeader, destreak); if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) { @@ -463,5 +455,5 @@ } // add keyword indicating that streaks have been removed - addDestreakKeyword(maskHeader); + addDestreakKeyword(maskHeader, destreak); if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", @@ -493,5 +485,5 @@ // add keyword indicating that streaks have been removed - addDestreakKeyword(weightHeader); + addDestreakKeyword(weightHeader, destreak); if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", @@ -683,5 +675,5 @@ psVector *tiles) { - if (!sfile) { + if (!sfile || !sfile->write) { return; } @@ -861,4 +853,5 @@ streaksExit("", PS_EXIT_DATA_ERROR); } + sfile->fits = NULL; psFree(sfile->header); sfile->header = NULL; @@ -1198,4 +1191,10 @@ weight->data.F32[y][x] = NAN; } + } else if (weightType == PS_TYPE_F64) { + double weightVal = weight->data.F64[y][x]; + if (!isnan(weightVal)) { + ++nandWeights; + weight->data.F64[y][x] = NAN; + } } else if(weightType == PS_TYPE_S16) { double weightVal = weight->data.S16[y][x]; @@ -1249,4 +1248,6 @@ psMetadataAddU16(in->header, PS_LIST_TAIL, "ZBLANK", 0, "", 32767); } else if (in->image->type.type == PS_TYPE_F32) { + in->exciseValue = NAN; + } else if (in->image->type.type == PS_TYPE_F64) { in->exciseValue = NAN; } else { Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.h =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.h (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.h (revision 30447) @@ -5,5 +5,5 @@ sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect, - psString outputFilename, bool required); + psString outputFilename, bool required, bool checkIfDestreaked); void closeImages(streakFiles *sfiles); @@ -12,5 +12,5 @@ void readImage(sFile *sfile, int extnum, ippStage stage, bool isMask); -void copyPHU(streakFiles *sfiles, bool remove); +void copyPHU(streakFiles *sfiles, bool remove, bool destreaked); void copyTable(sFile *out, sFile *in, int extnum); void copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles); @@ -24,5 +24,5 @@ void readImageFrom_pmFile(streakFiles *sf); -void addDestreakKeyword(psMetadata *); +void addDestreakKeyword(psMetadata *, bool value); void addRecoveryKeyword(psMetadata *); Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksrelease.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksrelease.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksrelease.c (revision 30447) @@ -24,5 +24,5 @@ if (sfiles->stage == IPP_STAGE_RAW) { // copy PHU to output files - copyPHU(sfiles, true); + copyPHU(sfiles, false, true); // advance to the first image extension Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.c (revision 30447) @@ -93,5 +93,5 @@ // Raw files have a phu and multiple extensions, one per chip // Since this is a raw file, copy it's PHU to output files - copyPHU(sfiles, true); + copyPHU(sfiles, true, true); // advance to the first image extension @@ -275,17 +275,21 @@ psLogMsg("streaksremove", PS_LOG_INFO, "time to run streaksremove: %f\n", total_time); - if (sfiles->statsFile) { - const char *statsMDC = psMetadataConfigFormat(sfiles->stats); - if (!statsMDC || strlen(statsMDC) == 0) { - psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n"); - } else { - fprintf(sfiles->statsFile, "%s", statsMDC); - psFree(statsMDC); - fclose(sfiles->statsFile); - sfiles->statsFile = NULL; - psFree(sfiles->stats); - sfiles->stats = NULL; - } - } + psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS"); + + if (statsFileName) { + // Write out + psString resolved = pmConfigConvertFilename(statsFileName, config, true, true); // Resolved filename + if (!resolved) { + psError(psErrorCodeLast(), false, "Unable to resolve statistics file name"); + return false; + } + if (!psMetadataConfigWrite(sfiles->stats, resolved, NULL)) { + psError(psErrorCodeLast(), false, "Unable to serialize stats metadata.\n"); + psFree(resolved); + return false; + } + psFree(resolved); + } + // all done. Clean up to look for memory leaks. @@ -613,5 +617,5 @@ addRecoveryKeyword(sf->recImage->header); } - addDestreakKeyword(sf->outImage->header); + addDestreakKeyword(sf->outImage->header, true); if (!SFILE_IS_IMAGE(sf->inImage)) { @@ -639,5 +643,5 @@ addRecoveryKeyword(sf->recMask->header); } - addDestreakKeyword(sf->outMask->header); + addDestreakKeyword(sf->outMask->header, true); // Note: we don't excise the mask pixels even if exciseAll is true. setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, false); @@ -671,5 +675,5 @@ addRecoveryKeyword(sf->recWeight->header); } - addDestreakKeyword(sf->outWeight->header); + addDestreakKeyword(sf->outWeight->header, true); setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll); @@ -767,5 +771,5 @@ } } - } else { + } else if (sfiles->inImage->image->type.type == PS_TYPE_F32) { float imageValue = sfiles->inImage->image->data.F32[y][x]; if (sfiles->recImage && !isExciseValue(imageValue, sfiles->inImage->exciseValue) ) { @@ -783,12 +787,28 @@ } } + } else { + // We could handle F64 but I don't think we ever get them. + // Should catch this earlier + psError(PS_ERR_IO, true, "unexpected image type %x found", sfiles->inImage->image->type.type ); + streaksExit("", PS_EXIT_PROG_ERROR); } if (sfiles->outWeight) { - if (sfiles->recWeight) { - sfiles->recWeight->image->data.F32[y][x] = sfiles->inWeight->image->data.F32[y][x]; - } // Assume that weight images are always a floating point type - sfiles->outWeight->image->data.F32[y][x] = NAN; + if (sfiles->inWeight->image->type.type == PS_TYPE_F32) { + if (sfiles->recWeight) { + sfiles->recWeight->image->data.F32[y][x] = sfiles->inWeight->image->data.F32[y][x]; + } + sfiles->outWeight->image->data.F32[y][x] = NAN; + } else if (sfiles->inWeight->image->type.type == PS_TYPE_F64) { + if (sfiles->recWeight) { + sfiles->recWeight->image->data.F64[y][x] = sfiles->inWeight->image->data.F64[y][x]; + } + sfiles->outWeight->image->data.F64[y][x] = NAN; + } else { + // Should catch this earlier + psError(PS_ERR_IO, true, "unexpected weight image type %x found", sfiles->inWeight->image->type.type ); + streaksExit("", PS_EXIT_PROG_ERROR); + } } if (sfiles->outMask) { @@ -912,5 +932,5 @@ streaksExit("", PS_EXIT_DATA_ERROR); } - addDestreakKeyword(header); + addDestreakKeyword(header, true); if (!psFitsWriteBlank(out->fits, header, extname)) { @@ -973,5 +993,5 @@ psArrayRealloc(outTable, j); - addDestreakKeyword(header); + addDestreakKeyword(header, true); if (psArrayLength(outTable) > 0) { printf("Censored %d sources\n", numCensored); Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.h =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.h (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.h (revision 30447) @@ -18,4 +18,5 @@ psString name; bool inNebulous; + bool write; psFits *fits; int nHDU; Index: branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c =================================================================== --- branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c (revision 30446) +++ branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c (revision 30447) @@ -36,5 +36,5 @@ if (sfiles->stage == IPP_STAGE_RAW) { // copy PHU to output files - copyPHU(sfiles, false); + copyPHU(sfiles, false, false); // advance to the first image extension @@ -296,4 +296,5 @@ sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header); + addDestreakKeyword(sf->outImage->header, false); if (!SFILE_IS_IMAGE(sf->inImage)) { @@ -317,4 +318,6 @@ readImage(sf->recMask, sf->extnum, sf->stage, true); sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header); + + addDestreakKeyword(sf->outMask->header, false); // XXX: TODO @@ -329,4 +332,5 @@ sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header); + addDestreakKeyword(sf->outWeight->header, false); setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);