Changeset 42183 for branches/eam_branches/ipp-20220316

Timestamp:

Apr 28, 2022, 4:13:55 PM (4 years ago)

Author:

eugene

Message:

fpcamera now reads in the chip image, mask, variance; generates pmSources and loads them on the analysis MD; saves the residual image

Location:

branches/eam_branches/ipp-20220316/fpcamera/src

Files:

• Makefile.am (modified) (1 diff)
• fpcamera.c (modified) (1 diff)
• fpcamera.h (modified) (1 diff)
• fpcameraAnalysis.c (modified) (3 diffs)
• fpcameraArguments.c (modified) (2 diffs)
• fpcameraChooseRefstars.c (added)
• fpcameraDataSave.c (modified) (1 diff)
• fpcameraLoadRefstars.c (modified) (9 diffs)
• fpcameraParseCamera.c (modified) (1 diff)

branches/eam_branches/ipp-20220316/fpcamera/src/Makefile.am

@@ -18 +18 @@
         fpcameraReadAstrometry.c    \
         fpcameraLoadRefstars.c      \
+        fpcameraChooseRefstars.c    \
         fpcameraCleanup.c
 

branches/eam_branches/ipp-20220316/fpcamera/src/fpcamera.c

@@ -18 +18 @@
     fprintf (stderr, "USAGE: one of the following:\n\n");
     fprintf (stderr, "  fpcamera -file filename[,filename,...] -mask maskfile[,maskfile,...] -variance varfile[,varfile,...] -astrom-file (smffile) OutFileBaseName\n");
-    fprintf (stderr, "  fpcamera -list filelist -masklist masklist -varlist varlist -astrom-file (smffile) -ref (dvopath) -OutFileBaseName\n\n");
+    fprintf (stderr, "  fpcamera -list filelist -masklist masklist -varlist varlist -astrom-file (smffile) -OutFileBaseName\n\n");
     fprintf (stderr, "where:\n");
     fprintf (stderr, "  FileNameList is a text file containing filenames, one per line\n");

branches/eam_branches/ipp-20220316/fpcamera/src/fpcamera.h

@@ -49 +49 @@
 bool              fpcameraLoadRefstars (pmFPAfile *input, pmConfig *config);
 bool              fpcameraReadAstrometry (pmFPAfile *input, pmConfig *config);
+bool              fpcameraChooseRefstars (pmFPAfile *input, pmFPAfile *astrom, pmFPAview *view);
 
 // Return version strings.

branches/eam_branches/ipp-20220316/fpcamera/src/fpcameraAnalysis.c

@@ -1 +1 @@
 # include "fpcamera.h"
 
-# define ESCAPE(ERROR, MSG) { psErrorStackPrint(stderr, MSG); return false; }
+# define ESCAPE(ERROR, MSG) { psErrorStackPrint(stderr, MSG); psFree (view); return false; }
 
 /* \brief this function loops over chips and performs forced photometry for the references */
@@ -7 +7 @@
 
     bool status;
+    pmChip *chip;
+    pmCell *cell;
+    pmReadout *readout;
+    pmFPAview *view = NULL;
 
     // measure the total elapsed time in fpcameraAnalysis.
@@ -15 +19 @@
     if (!recipe) ESCAPE (FPCAMERA_ERR_CONFIG, "Can't find FPCAMERA recipe");
 
-    psLogMsg("fpcamera", 3, "TIMEMARK: fpcameraMosaicAstrom: %f sec\n", psTimerMark ("complete"));
+    // loop over the input images
+    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "FPCAMERA.INPUT");
+    if (!input) ESCAPE(FPCAMERA_ERR_CONFIG, "Can't find or interpret output file rule FPCAMERA.INPUT!");
+
+    // astrometry reference (smf)
+    pmFPAfile *astrom = psMetadataLookupPtr (NULL, config->files, "FPCAMERA.INPUT.ASTROM");
+    if (!astrom) ESCAPE(FPCAMERA_ERR_CONFIG, "Can't find or interpret output file rule FPCAMERA.INPUT.ASTROM!");
+
+    // only activate input image-type files
+    pmFPAfileActivate (config->files, false, NULL);
+    pmFPAfileActivate (config->files, true, "FPCAMERA.INPUT");
+    pmFPAfileActivate (config->files, true, "FPCAMERA.INPUT.MASK");
+    pmFPAfileActivate (config->files, true, "FPCAMERA.INPUT.VARIANCE");
+    pmFPAfileActivate (config->files, true, "FPCAMERA.RESID");
+    // Note FPCAMERA.RESID is tied to FPCAMERA.INPUT so they must be active in the same block.
+
+    view = pmFPAviewAlloc (0);
+
+    // load images at FPA level (if appropriate)
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE(FPCAMERA_ERR_DATA, "failed to load images at FPA level");
+
+    // load images at chip level (if appropriate) 
+    while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) {
+        psTrace ("fpcamera", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        if (!chip->process || !chip->file_exists) { continue; }
+        if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE(FPCAMERA_ERR_DATA, "failed to load images at Chip level");
+
+        // loop over all cells in chip
+        while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) {
+            psLogMsg ("fpcamera", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+
+            // loop over all readouts in cell
+            while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) {
+                psLogMsg ("fpcamera", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+                if (!readout->data_exists) { continue; }
+
+                fpcameraChooseRefstars (input, astrom, view);
+            }
+        }
+        if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE(FPCAMERA_ERR_IO, "failure in IOChecks(AFTER) at Chip");
+    }
+    if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE(FPCAMERA_ERR_IO, "failure in IOChecks(AFTER) at FPA");
+
+    psFree (view);
+
+    psLogMsg("fpcamera", 3, "TIMEMARK: fpcameraAnalysis: %f sec\n", psTimerMark ("analysis"));
     return true;
 }
+
+// NOTES
+// chip->process is set (unset) based on command-line -chip selections (in fpcameraArguments)
+// chip->file_exists is set (in pmFPAFlags.c:pmChipSetFileStatus) by pmFPAfileDefineFromArgs (in pmFPAAddSource...)

branches/eam_branches/ipp-20220316/fpcamera/src/fpcameraArguments.c

@@ -50 +50 @@
     }
 
+    // chip selection is used to limit chips to be processed
+    if ((N = psArgumentGet (argc, argv, "-save-resid"))) {
+        psArgumentRemove (N, &argc, argv);
+        psMetadataAddBool (config->arguments, PS_LIST_TAIL, "SAVE.RESID", PS_META_REPLACE, "", true);
+    }
+
     // specify the input images to process
     status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT",    "-file",     "-list");
@@ -61 +67 @@
     status = pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-varlist");
     if (!status) ESCAPE(FPCAMERA_ERR_ARGUMENTS, "Missing -variance (input) or -varlist (input)");
-
-    // specify the reference data source (DVO format or ??)
-    // N = psArgumentGet (argc, argv, "-ref");
-    // if (!N) ESCAPE (FPCAMERA_ERR_ARGUMENTS, "Missing -ref (database)");
-    // psArgumentRemove (N, &argc, argv);
-    // psMetadataAddStr (config->arguments, PS_LIST_TAIL, "FPCAMERA.CATDIR", PS_META_REPLACE, "", argv[N]);
-    // psArgumentRemove (N, &argc, argv);
 
     // specify the astrometry calibration

branches/eam_branches/ipp-20220316/fpcamera/src/fpcameraDataSave.c

@@ -24 +24 @@
     pmFPAfileActivate (config->files, true, "FPCAMERA.OUTPUT");
     pmFPAfileActivate (config->files, true, "FPCAMERA.INPUT.ASTROM");
+    // Note: I/O for the image-type files is performed in fpcameraAnalysis
 
     view = pmFPAviewAlloc (0);

branches/eam_branches/ipp-20220316/fpcamera/src/fpcameraLoadRefstars.c

@@ -1 +1 @@
 # include "fpcamera.h"
-# define ELIXIR_MODE 1
 
 # define ESCAPE(ERROR,...) { p_psError(__FILE__,__LINE__,__func__,ERROR,false,__VA_ARGS__); return false; }
@@ -37 +36 @@
     DECmax += dDEC * fieldPadding;
 
-    // XXX: fpcameraArguments.c looks for this explicitly on the command line, but stores it in arguments.
-    // this could be a recipe value which can also be defined as a command-line option 
-
     // grab the FPCAMERA.CATDIR name from the FPCAMERA recipe
     char *catdir_recipe = psMetadataLookupStr(&status, recipe, "FPCAMERA.CATDIR");
     if (!catdir_recipe) ESCAPE(FPCAMERA_ERR_CONFIG, "Need a recipe for the catdir!");
 
-    // substitute abstract name with concrete name, if present in FPCAMERA.CATDIRS
-    psMetadata *catdirs = psMetadataLookupMetadata(&status, config->site, "FPCAMERA.CATDIRS"); // List of cameras
-
-    // XXX it is allowed that FPCAMERA.CATDIRS is not defined in site.config: the value must be a real path
-    // if (!catdirs) ESCAPE(FPCAMERA_ERR_CONFIG, "Unable to find FPCAMERA.CATDIRS in the system configuration.\n");
+    // *** substitute abstract name with concrete name, if present in FPCAMERA.CATDIRS ***
 
     // the name in the recipe may be one of:
@@ -54 +46 @@
     // (B) a reference to the name in the FPCAMERA.CATDIRS folder in site.config
     // (C) a reference to a folder in the FPCAMERA.CATDIRS folder in site.config, containing multiple copy locations
+
+    // Folder in site.config containing a list of FPCAMERA.CATDIR entries.
+    // NOTE: it is allowed that FPCAMERA.CATDIRS not be defined in site.config: then FPCAMERA.CATDIR must be a real path
+    psMetadata *catdirs = psMetadataLookupMetadata(&status, config->site, "FPCAMERA.CATDIRS"); // List of cameras
 
     // can we find a plain string matching catdir_recipe in the catdirs folder?
@@ -106 +102 @@
     // supply a known output format (for CATALOG output) so the code below knows what to read
     // XXX I do not think this affects the getstar command (CATFORMAT is used to define the dvo db format)
-    if (ELIXIR_MODE) {
-        psStringAppend (&getstarCommand, " -D CATMODE mef -D CATFORMAT elixir");
-    } else {
-        psStringAppend (&getstarCommand, " -D CATMODE mef -D CATFORMAT panstarrs");
-    }
+    // if (ELIXIR_MODE) {
+    //     psStringAppend (&getstarCommand, " -D CATMODE mef -D CATFORMAT elixir");
+    // } else {
+    //     psStringAppend (&getstarCommand, " -D CATMODE mef -D CATFORMAT panstarrs");
+    // }
 
-    // check for default name (use .ptolemyrc), or use specified CATDIR
-    if (strcasecmp(CATDIR, "NONE")) {
-        psStringAppend (&getstarCommand, " -D CATDIR %s", CATDIR);
-    }
+    // Define the full getstar command.  Above, we require CATDIR to be valid and defined at
+    // this point.  Also add region and output filename.
+    psStringAppend (&getstarCommand, " -D CATDIR %s", CATDIR);
+    psStringAppend (&getstarCommand, " -format %s", outformat);
+    psStringAppend (&getstarCommand, " -region %f %f %f %f -o %s", RAmin, DECmin, RAmax, DECmax, tempFile);
+
     psFree(CATDIR);
-
-    psStringAppend (&getstarCommand, " -format %s", outformat);
-
-    // add region and output filename
-    psStringAppend (&getstarCommand, " -region %f %f %f %f -o %s", RAmin, DECmin, RAmax, DECmax, tempFile);
-    psTrace ("fpcamera", 3, "%s\n", getstarCommand);
 
     psLogMsg("fpcamera", PS_LOG_INFO, "getstar command: %s", getstarCommand);
 
-    // XXX use psPipe: catch stderr, stdout, allow for Nsec timeout...
-    // use fork to add timeout capability
+    // run getstar, result is saved in the temp file
     status = system (getstarCommand);
     if (status) ESCAPE(FPCAMERA_ERR_REFSTARS, "error loading reference data");
@@ -140 +131 @@
 
     psArray *refstars = NULL;
-    if (!strcmp (outformat, "CATALOG")) {
-      refstars = fpcameraReadGetstarCatalog (fits);
-    }
     if (!strcmp (outformat, "PS1_DEV_0")) {
       refstars = fpcameraReadGetstar_PS1_DEV_0 (fits);
@@ -160 +148 @@
     unlink (tempFile);
 
-# if (0)
-
-    // dump or plot the available refstars
-    if (psTraceGetLevel("fpcamera.dump") > 0) {
-        fpcameraDumpRefstars (refstars, "refstars.dat");
-    }
-
-    pmAstromVisualPlotRefStars (refstars, recipe);
-
-    if (psTraceGetLevel("fpcamera.plot") > 0) {
-        fpcameraPlotRefstars (refstars, recipe);
-    }
-# endif
-    
     psMetadataAdd (input->fpa->analysis, PS_LIST_TAIL, "FPCAMERA.REFSTARS", PS_DATA_ARRAY, "reference sources", refstars);
     psFree (refstars);
@@ -180 +154 @@
 }
 
-psArray *fpcameraReadGetstarCatalog (psFits *fits) {
-
-    bool status;
-
-    if (ELIXIR_MODE) {
-        psFitsMoveExtName (fits, "DVO_AVERAGE_ELIXIR");
-    } else {
-        psFitsMoveExtName (fits, "DVO_AVERAGE_PANSTARRS");
-    }
-
-    long numSources = psFitsTableSize(fits); // Number of sources in table
-
-    // convert the Average table to the pmAstromObj entries
-    psArray *refstars = psArrayAllocEmpty (numSources);
-    for (int i = 0; i < numSources; i++) {
-        pmAstromObj *ref = pmAstromObjAlloc ();
-
-        psMetadata *row = psFitsReadTableRow(fits, i); // Table row
-
-        // DVO tables are stored in degrees
-        if (ELIXIR_MODE) {
-            ref->sky->r   = RAD_DEG*psMetadataLookupF32 (&status, row, "RA");
-            ref->sky->d   = RAD_DEG*psMetadataLookupF32 (&status, row, "DEC");
-            ref->Mag      = 0.001*psMetadataLookupS32 (&status, row, "MAG");  // ELIXIR uses millimags
-            ref->Color    = 0.0;
-        } else {
-            ref->sky->r   = RAD_DEG*psMetadataLookupF64 (&status, row, "RA");
-            ref->sky->d   = RAD_DEG*psMetadataLookupF64 (&status, row, "DEC");
-            ref->Mag      = psMetadataLookupF32 (&status, row, "MAG"); // PANSTARRS uses mags
-            ref->Color    = 0.0;
-        }
-        ref->magCal   = ref->Mag;
-
-        // XXX VERY temporary hack to avoid M31 bulge
-        if ((fabs(ref->sky->r - 0.186438) < 0.002) && (fabs(ref->sky->d - 0.720270) < 0.002)) {
-          psFree (ref);
-          psFree (row);
-          continue;
-        }
-
-        psArrayAdd (refstars, 100, ref);
-        psFree (ref);
-        psFree (row);
-    }
-    return refstars;
-}
-
+// method to read PS1_DEV_0 format
+// Note other options can be found in psastroLoadRefstars.c
 psArray *fpcameraReadGetstar_PS1_DEV_0 (psFits *fits) {
 
@@ -250 +179 @@
             ref->Color = MagC1 - MagC2;
         } else {
-            // XXX save the color and the slope in the table header?
             ref->Color = 0.0;
         }
@@ -269 +197 @@
 }
 
-# undef ESCAPE
 
-# if (0)
-
-# define ESCAPE(MSG) { \
-  psLogMsg ("fpcamera", PS_LOG_INFO, MSG); \
-  goto escape; }
-
-char *fpcameraSetMagLimit (float *minMag, float *maxRho, pmConfig *config, const char *source) {
-
-    bool status;
-    char *photcode;
-
-    // select the current recipe
-    psMetadata *recipe  = psMetadataLookupPtr (NULL, config->recipes, FPCAMERA_RECIPE);
-
-    // select the input data sources
-    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, source);
-    if (!input) {
-        psLogMsg ("fpcamera", PS_LOG_DETAIL, "no supplied reference header data");
-        photcode = psStringCopy ("NONE");
-        return photcode;
-    }
-    assert (input->fpa);
-
-    *maxRho = psMetadataLookupF32(&status, recipe, "DVO.GETSTAR.MAX.RHO");
-    if (!status) {
-        psError(FPCAMERA_ERR_CONFIG, false, "DVO.GETSTAR.MAX.RHO missing from recipe");
-        return NULL;
-    }
-
-    // select the filter; default to fixed photcode and mag limit otherwise
-    char *filter = psMetadataLookupStr (&status, input->fpa->concepts, "FPA.FILTERID");
-    if (!status) ESCAPE ("missing FPA.FILTER in concepts");
-
-    float exptime = psMetadataLookupF32 (&status, input->fpa->concepts, "FPA.EXPOSURE");
-    if (!status) ESCAPE ("missing FPA.EXPOSURE in concepts");
-
-    // we need to select the PHOTCODE.DATA folder that matches our filter
-    psMetadataItem *item = psMetadataLookup (recipe, "PHOTCODE.DATA");
-    if (!item) ESCAPE ("PHOTCODE.DATA folders missing");
-    if (item->type != PS_DATA_METADATA_MULTI) ESCAPE ("PHOTCODE.DATA not a multi");
-
-    float minInst = psMetadataLookupF32(&status, recipe, "DVO.GETSTAR.MIN.MAG.INST");
-    if (!status) ESCAPE ("missing DVO.GETSTAR.MIN.MAG.INST");
-
-    // if non zero override the zero point in the PHOTCODE.DATA with this value
-    float fixedzeropt = psMetadataLookupF32(&status, recipe, "DVO.GETSTAR.FIXED.ZEROPT");
-
-    // PHOTCODE.DATA is a multi of metadata items
-    psListIterator *iter = psListIteratorAlloc(item->data.list, PS_LIST_HEAD, false);
-
-    psMetadataItem *refItem = NULL;
-    while ((refItem = psListGetAndIncrement (iter))) {
-        if (refItem->type != PS_DATA_METADATA) ESCAPE ("PHOTCODE.DATA entry is not a metadata folder");
-
-        char *refFilter = psMetadataLookupStr (&status, refItem->data.md, "FILTER");
-        if (!status) {
-            psLogMsg ("fpcamera", PS_LOG_INFO, "a PHOTCODE.DATA recipe folder is missing FILTER");
-            continue;
-        }
-
-        // does this entry match the current filter?
-        if (strcmp (refFilter, filter)) continue;
-
-        psLogMsg ("fpcamera", PS_LOG_DETAIL, "PHOTCODE.DATA found for filter %s", filter);
-
-        float zeropt = psMetadataLookupF32 (&status, refItem->data.md, "ZEROPT");
-        if (!status) {
-            psLogMsg ("fpcamera", PS_LOG_INFO, "a PHOTCODE.DATA recipe folder is missing FILTER");
-            continue;
-        }
-        photcode = psMetadataLookupStr (&status, refItem->data.md, "PHOTCODE");
-        if (!status) {
-            psLogMsg ("fpcamera", PS_LOG_INFO, "a PHOTCODE.DATA recipe folder is missing FILTER");
-            continue;
-        }
-        if (fixedzeropt != 0.0) {
-            // override the recipe's zero point with the fixed value (used for stacks)
-            zeropt = fixedzeropt;
-        }
-
-        // convert the minInst to a calibrated minimum magnitude
-        *minMag = minInst + 2.5*log10(exptime) + zeropt;
-
-        psFree (iter);
-        return photcode;
-    }
-    psFree (iter);
-
-  escape:
-    photcode = psMetadataLookupStr(NULL, recipe, "DVO.GETSTAR.PHOTCODE");
-    PS_ASSERT (photcode, NULL);
-
-    // give up and use fixed value
-    *minMag = psMetadataLookupF32(NULL, recipe, "DVO.GETSTAR.MIN.MAG");
-    return photcode;
-}
-# endif

branches/eam_branches/ipp-20220316/fpcamera/src/fpcameraParseCamera.c

@@ -34 +34 @@
     output->save = true;
 
+    // optionally save the residual image
+    if (psMetadataLookupBool(&status, config->arguments, "SAVE.RESID")) {
+        pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, input, "FPCAMERA.RESID");
+        if (!output) ESCAPE(FPCAMERA_ERR_CONFIG, "Cannot find a rule for FPCAMERA.RESID");
+        output->save = true;
+    }
+
     // Chip selection: turn on only the chips specified (option is not required)
     char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS");