Index: /tags/ipp-ops-20220906/ppSub/src/ppSubCamera.c =================================================================== --- /tags/ipp-ops-20220906/ppSub/src/ppSubCamera.c (revision 42931) +++ /tags/ipp-ops-20220906/ppSub/src/ppSubCamera.c (revision 42932) @@ -119,6 +119,10 @@ -// Define an output file that will be used in a calculation -// This means it might already be available in the RUN metadata +// Define an output file that will be used in a calculation. This means it might already be +// available in the RUN metadata + +// EAM 2022.10.14 : If the file is not found, treat it as if it did not exist in the input +// RUN metadata. Currently, this is only used in this file to select +// PPSUB.OUTPUT.KERNELS. static pmFPAfile *defineCalcFile(pmConfig *config, // Configuration pmFPAfile *bind, // File to which to bind, or NULL @@ -133,6 +137,11 @@ pmFPAfile *file = pmFPAfileDefineFromRun(&status, NULL, config, filerule); // File to return if (!status) { - psError(psErrorCodeLast(), false, "Failed to load file definition for %s", filerule); - return NULL; + bool requireSubkernel = psMetadataLookupBool(NULL, config->arguments, "-require-subkernel"); + if (requireSubkernel) { + psError(psErrorCodeLast(), false, "Failed to load file definition for %s", filerule); + return NULL; + } else { + psWarning("Failed to load file definition for %s, regenerate", filerule); + } } file = pmFPAfileBindFromArgs(&status, bind, config, filerule, argname); @@ -171,4 +180,68 @@ } +bool pmConfigFixSkycellCamera (pmConfig *config) { + + // We have some static MDC files used for update for which we are missing PSCAMERA. + // (This was an attempt to address cross-camera diffs, but was only a partial solution) + // Here we need to check for -SKYCELL entries and re-instate the PSCAMERA entries + + bool mdok = false; + psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->system, "CAMERAS"); + psAssert(cameras, "missing cameras in system config info"); + + // iterate over the cameras and find ones with names like _*-SKYCELLS + // psMetadataIterator *camerasIter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, "^_.+-SKYCELL$"); + psMetadataIterator *camerasIter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, NULL); + psAssert(camerasIter, "unable to generate iterator?"); + + psMetadataItem *camerasItem = NULL; // Item from the metadata + while ((camerasItem = psMetadataGetAndIncrement(camerasIter))) { + psAssert(camerasItem->type == PS_DATA_METADATA, "camerasItem has invalid type"); + + char *name = camerasItem->name; + psAssert (name, "NULL name for item"); + + int nameLen = strlen(name); + if (name[0] != '_') continue; + if (nameLen <= 9) continue; + char *p = &name[nameLen - 8]; + if (strcmp(p, "-SKYCELL")) continue; + + // we have found a -SKYCELL camera. does it already have PSCAMERA? + + // remove PSCAMERA entries from *-SKYCELL cameras + psMetadata *formats = psMetadataLookupMetadata(&mdok, camerasItem->data.md, "FORMATS"); // List of formats + psAssert (formats, "missing FORMATS in camera.config "); + + psMetadata *format = psMetadataLookupMetadata(&mdok, formats, "SKYCELL"); // one true format for skycells + psAssert (format, "missing SKYCELL metaformat for -SKYCELL entry "); + + psMetadata *rule = psMetadataLookupMetadata(&mdok, format, "RULE"); // one true format for skycells + psAssert (rule, "missing RULE in SKYCELL metaformat for -SKYCELL entry "); + + psString pscamera = psMetadataLookupStr(&mdok, rule, "PSCAMERA"); // one true format for skycells + if (pscamera) continue; // PSCAMERA exists, we are good + + // the name of this camera is encoded in the portion of the word skipped above + // name[1] to name[nameLen - 8] : number of bytes to copy is : nameLen - 8 - 1: + + // _GPC2-SKYCELL + // 0123456789012 + // nameLen = 13 + // nameLen - 8 - 1 = 4 + char cameraName[64]; + + int nByte = nameLen - 8 - 1; + psAssert (nByte < 64 - 1, "camera name is too long"); + + strncpy (cameraName, &name[1], nByte); cameraName[nByte] = 0; + + psMetadataAddStr (rule, PS_LIST_TAIL, "PSCAMERA", PS_META_REPLACE, "", cameraName); + } + psFree (camerasIter); + + return true; +} + pmConfig *pmConfigMakeTemp (pmConfig *config) { pmConfig *altconfig = pmConfigAlloc(); @@ -177,5 +250,7 @@ altconfig->user = psMemIncrRefCounter(config->user); // inherit from primary camera altconfig->site = psMemIncrRefCounter(config->site); // inherit from primary camera - altconfig->system = psMemIncrRefCounter(config->system); // inherit from primary camera + + psFree (altconfig->system); + altconfig->system = psMetadataCopy(NULL, config->system); // container for camera-specific recipe values (to be dropped) psFree (altconfig->files); @@ -188,8 +263,7 @@ altconfig->recipes = psMetadataCopy(NULL, config->recipes); // container for camera-specific recipe values (to be dropped) -# if (0) - // XXX ppStack seems to work without removing PSCAMERA - // remove PSCAMERA entries from *-SKYCELL cameras - // XXX make this optional to force matching cameras if desired? + // remove PSCAMERA entries from *-SKYCELL cameras. This allows skycell images from + // one camera to match those of another camera. XXX Make this optional to force + // matching cameras if desired? bool mdok = false; psMetadata *cameras = psMetadataLookupMetadata(&mdok, altconfig->system, "CAMERAS"); @@ -229,7 +303,23 @@ psMetadataRemoveKey (rule, "PSCAMERA"); // allow any camera skycell to match } -# endif + psFree (camerasIter); return (altconfig); +} + +bool ppSubCopyPSCamera (pmFPAfile *tgt, pmFPAfile *src) { + + bool success; + psMetadata *ruleSrc = psMetadataLookupMetadata(&success, src->format, "RULE"); // one true format for skycells + psAssert (ruleSrc, "missing RULE in src->format"); + + psString pscameraSrc = psMetadataLookupStr(&success, ruleSrc, "PSCAMERA"); // one true format for skycells + psAssert (pscameraSrc, "missing PSCAMERA in src file"); + + psMetadata *ruleTgt = psMetadataLookupMetadata(&success, tgt->format, "RULE"); // one true format for skycells + psAssert (ruleTgt, "missing RULE in tgt->format"); + + psMetadataAddStr (ruleTgt, PS_LIST_TAIL, "PSCAMERA", PS_META_REPLACE, "", pscameraSrc); + return true; } @@ -241,4 +331,6 @@ pmConfig *config = data->config; psAssert(config, "Require configuration"); + + pmConfigFixSkycellCamera (config); // Input image @@ -267,28 +359,32 @@ } + int did_rewrite = 0; + // Use a temporary config for the reference image, keeping the main user, site, // system, files, arguments entries. This allows the reference image to be from a // different camera than the input image. NOTE: there is no check that these two // images match in terms of size, pixel scale, etc. That is up to the user. + // The temporary config structure has PSCAMERA entries removed from the SKYCELL rules + // to allow subtraction between cameras. pmConfig *refconfig = pmConfigMakeTemp(config); -# if (0) - // EAM TEST: can we bind the ref to the input to force the right skycell format? - // Reference image - pmFPAfile *ref = defineInputFile(&success, config, input, "PPSUB.REF", "REF", PM_FPA_FILE_IMAGE); - if (!success) { - psError(psErrorCodeLast(), false, "Failed to build FPA from PPSUB.REF"); - return false; - } -# else // Reference image pmFPAfile *ref = defineInputFile(&success, refconfig, NULL, "PPSUB.REF", "REF", PM_FPA_FILE_IMAGE); if (!success) { psError(psErrorCodeLast(), false, "Failed to build FPA from PPSUB.REF"); - return false; - } -# endif - + + psErrorClear(); + fprintf(stderr,"%s\n", config->cameraName); + psStringSubstitute(&( config->cameraName ),"GPC1","GPC2"); + fprintf(stderr,"%s\n", config->cameraName); + did_rewrite = 1; + + ref = defineInputFile(&success, config, NULL, "PPSUB.REF", "REF", PM_FPA_FILE_IMAGE); + if (!success) { + return false; + } + } + // Reference mask defineInputFile(&success, refconfig, ref, "PPSUB.REF.MASK", "REF.MASK", PM_FPA_FILE_MASK); if (!success) { @@ -296,5 +392,5 @@ return false; } - + // Reference variance pmFPAfile *refVar = defineInputFile(&success, refconfig, ref, "PPSUB.REF.VARIANCE", "REF.VARIANCE", PM_FPA_FILE_VARIANCE); if (!success) { @@ -302,4 +398,6 @@ return false; } + // copy ref->format(RULE) PSCAMERA from input->format(RULE) + ppSubCopyPSCamera (ref, input); psFree (refconfig); @@ -308,10 +406,35 @@ pmConfig *srcconfig = pmConfigMakeTemp(config); - defineInputFile(&success, srcconfig, NULL, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF); + // Sources input file (CMF) + pmFPAfile *src = defineInputFile(&success, srcconfig, NULL, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF); if (!success) { + if (did_rewrite == 1) { + // Sometimes we use the warp sources instead of the stack + psErrorClear(); + did_rewrite = 0; + + fprintf(stderr,"%s\n", config->cameraName); + psStringSubstitute(&( config->cameraName ),"GPC2","GPC1"); + fprintf(stderr,"%s\n", config->cameraName); + defineInputFile(&success, config, NULL, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF); + if (!success) { + psError(psErrorCodeLast(), false, "Failed to build FPA from PPSUB.REF.SOURCES"); + return false; + } + } + else { psError(psErrorCodeLast(), false, "Failed to build FPA from PPSUB.REF.SOURCES"); return false; - } + } + } + // copy src->format(RULE) PSCAMERA from input->format(RULE) + ppSubCopyPSCamera (src, input); psFree (srcconfig); + + if (did_rewrite == 1) { + fprintf(stderr,"%s\n", config->cameraName); + psStringSubstitute(&( config->cameraName ),"GPC2","GPC1"); + fprintf(stderr,"%s\n", config->cameraName); + } // Now that the camera has been determined, we can read the recipe @@ -481,15 +604,38 @@ checkFileruleFileSave(jpeg3, config); +# if (0) + // XXX NOTE EAM 20240209: This block of code attempts to correct the problem of missing + // PSCAMERA entries, in this case in the output kernel definition. This seems to work, but + // there was a crash related to running psphot in update mode. This block is probably not + // responsible for the crash, but is commented out for the moment + + // The reference sources may not be from the same origin as the reference image, so + // use another temporary camera. + pmConfig *kernel_config = pmConfigMakeTemp(config); + // Output subtraction kernel - pmFPAfile *kernel = defineCalcFile(config, output, "PPSUB.OUTPUT.KERNELS", "KERNEL", - PM_FPA_FILE_SUBKERNEL); + pmFPAfile *kernel = defineCalcFile(kernel_config, output, "PPSUB.OUTPUT.KERNELS", "KERNEL", PM_FPA_FILE_SUBKERNEL); if (!kernel) { - psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to define file PPSUB.OUTPUT.KERNELS"); - return false; - } + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to define file PPSUB.OUTPUT.KERNELS"); + return false; + } + + // copy kernel->format(RULE) PSCAMERA from input->format(RULE) + ppSubCopyPSCamera (kernel, input); + psFree (kernel_config); +# else + // XXX NOTE EAM 20240209: This block of code matches the older version used in ipp-ps1-20220906-gentoo + + // Output subtraction kernel + pmFPAfile *kernel = defineCalcFile(config, output, "PPSUB.OUTPUT.KERNELS", "KERNEL", PM_FPA_FILE_SUBKERNEL); + if (!kernel) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to define file PPSUB.OUTPUT.KERNELS"); + return false; + } +# endif // psPhot input if (data->photometry || 1) { - psphotModelClassInit(); // load implementation-specific models + psphotModelClassInit(); // load implementation-specific models pmFPAfile *psphot = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT");