Changeset 42120

Timestamp:

Mar 17, 2022, 7:09:13 AM (4 years ago)

Author:

eugene

Message:

fixes to handle inconsistent numbers of chip files and smf entries; handle cases where there are no matches with the reference catalog

Location:

tags/ipp-ops-20220303-rc/ippTests/ippCheck

Files:

• . (modified) (1 prop)
• cammask.pro (modified) (5 diffs)
• camstats.pro (modified) (4 diffs)
• diffmask.pro (modified) (1 diff)
• ippCheck (modified) (2 diffs)
• warpmask.pro (modified) (1 diff)
• warpstats.pro (modified) (3 diffs)

tags/ipp-ops-20220303-rc/ippTests/ippCheck/cammask.pro

@@ -52 +52 @@
  $smfreal = `neb-locate -p $smffile`
  
+ # sets the list chipName:0 - chipName:n based on chipRun 
  get.chip.metadata $dbname $chipID
+
+ # set the list camMask:0 - camMask:n based on nebulous listing
+ # some cam-stage masks may be missing, these are set to NONE
+ # the number of entries in both lists is the same (chipName:n)
  get.chip.mask.files $camPath
  
+ # read in the PHU header
+ rd PHU $smfreal
+
+ # set the list chipNseq:0 - chipNseq:n based on smf entries.
+ # Some chips may be missing from the smf, these are set to -1
+ # $chipNseq:n = $chipName:n
+ get.chip.smf.Nseq $smfreal
+
  for i 0 $camMask:n
-   echo $chipName:$i $camMask:$i
+   echo $chipName:$i $chipNseq:$i $camMask:$i
  end
 
@@ -65 +78 @@
  if ($table:Ny == 0)
   echo "ERROR: no reference sources found in $refsrcs"
+
+  # generate an empty output file
+  $OUT_FIELDS = 
+  foreach field $SMF_FIELDS_CAM $REF_FIELDS_CAM $XTR_FIELDS_CAM
+    $OUT_FIELDS = $OUT_FIELDS $field\_out
+    create -empty $field\_out
+  end
+  
+  sprintf output "%s/%s.%s.cam.stf" $OUTDIR $expname $OUTVERSION
+  write -fits DATA $output $OUT_FIELDS
+  
+  # save this name for warp and diff analysis, if desired
+  $CAMSOURCE = $output
+
   break
  end
@@ -71 +98 @@
  set Nref = ramp(Rref)
 
- # read in the PHU header
- rd PHU $smfreal
-
- # create the chipName list from the smffile and calculate the index numbers.
- # NOTE: if a chip is missing from the smf, this may be inconsistent with chipMask:n
- # XXX for now, we fail is this happens
- list chipsRaw -x "ftable -list $smfreal | grep -n hdr | prcol 1 | sed s/.hdr//"
- for i 0 $chipsRaw:n
-  list word -splitbychar : $chipsRaw:$i
-  $chipNseq:$i = $word:0 - 3; # this number depends on the number of leading tables 
-  if ("$chipName:$i" != "$word:1")
-    echo "ERROR: missing chip from smffile: " $chipName:$i
-    break
-  end
- end
-
-  # delete any existing output vectors
-  foreach field $SMF_FIELDS_CAM $REF_FIELDS_CAM $XTR_FIELDS_CAM
-    delete -q $field\_out
-  end
+ # delete any existing output vectors
+ foreach field $SMF_FIELDS_CAM $REF_FIELDS_CAM $XTR_FIELDS_CAM
+   delete -q $field\_out
+ end
 
  # find matching entries in smf file 
  for i 0 $chipName:n
+   ## echo $chipName:$i $chipNseq:$i $camMask:$i
+
+   if ($chipNseq:$i < 0) continue
+
    get.chip.detect.stats $chipName:$i $chipNseq:$i
-   get.chip.mask.stats $camMask:$i
+
+   if ("$camMask:$i" == "NONE") 
+     # if the mask files are missing, all are masked or unmasked?
+     set Q1ref_onchp = zero(Xref_onchp) + 1.0
+   else
+     get.chip.mask.stats $camMask:$i
+   end
 
    foreach field $SMF_FIELDS_CAM
@@ -241 +262 @@
 
  # get the full list from nebulous (faster than getting each one at a time)
- list tmpName -x "neb-ls $pathbase.%.mk.fits"
+ list tmpName -x "neb-ls -c $pathbase.%.mk.fits"
 
  # the list of masks might not match the list of chips
+ # mark missing cam-stage masks with NONE
  $j = 0
+ echo "chipName:n $chipName:n"
  for i 0 $chipName:n
+   # if we get to the end of the list, assign the missing entries to NONE
+   if ($j >= $tmpName:n)
+    $camMask:$i = NONE
+    continue
+   end
    list word -splitbychar . $tmpName:$j
    $Nch = $word:n - 3
@@ -254 +282 @@
    end
    $camMask:$i = $tmpName:$j
-   if ($VERBOSE) echo "found chipMask $camMask:$i"
+   if ($VERBOSE) echo "found chipMask $camMask:$i ($chipName:$i) : $i $j $chipName:n $tmpName:n"
    if ($COPYMASKS) exec neb-copy $camMask:$i $OUTDIR
    $j ++
- end
- $camMask:n = $chipName:n
+   if ($j >= $tmpName:n) last
+ end
+ $camMask:n = $tmpName:n
+end
+
+# get the file names of the chip mask files.
+# these are actually the masks from the camera stage which
+# has the dynamic masking applied
+macro get.chip.smf.Nseq
+ if ($0 != 2)
+  echo "USAGE: get.camera.smf.Nseq (smffile)"
+  echo " sets chipNseq:0 - chipNseq:n"
+  break
+ end
+
+ local i j smffile
+ $smffile = $1
+
+ # identify the chips in the smffile and calculate the index numbers.
+ # NOTE: if a chip is missing from the smf, set Nseq to -1
+ list chipsRaw -x "ftable -list $smfreal | grep -n hdr | prcol 1 | sed s/.hdr//"
+
+ $j = 0
+ for i 0 $chipName:n
+   # if we get to the end of the list, assign the missing entries to NONE
+   if ($j >= $chipsRaw:n)
+    $chipNseq:$i = -1
+    continue
+   end
+   list word -splitbychar : $chipsRaw:$j
+   $myNseq = $word:0 - 3; # this number depends on the number of leading tables 
+   $mychip = $word:1
+   if ($mychip != $chipName:$i) 
+    $chipNseq:$i = -1
+    continue
+   end
+   $chipNseq:$i = $myNseq
+   if ($VERBOSE) echo "found chip entry $chipNseq:$i ($chipName:$i) : $i $j $chipName:n $chipsRaw:n"
+   $j ++
+ end
+ $chipNseq:n = $chipName:n
 end
 

tags/ipp-ops-20220303-rc/ippTests/ippCheck/camstats.pro

@@ -38 +38 @@
    # results in magFound, fracFound, numFound
    calc.camstats
-   if ($i == 0)
+   if (numTotal_sum?[] == 0)
      set numTotal_sum = numTotal
      set numFound_sum = numFound
@@ -57 +57 @@
  end
    
- set numTotal = numTotal_sum
- set numFound = numFound_sum
- set numPoss  = numPoss_sum
- set numGood  = numGood_sum
- delete numTotal_sum numFound_sum numPoss_sum numGood_sum
+ if (numTotal_sum?[] == 0)
+   calc.camstats 
+ else 
+   set numTotal = numTotal_sum
+   set numFound = numFound_sum
+   set numPoss  = numPoss_sum
+   set numGood  = numGood_sum
+   delete numTotal_sum numFound_sum numPoss_sum numGood_sum
+ end
 
  set fracFound = (numTotal > 0) ? numFound / numTotal : zero(numTotal)
@@ -79 +83 @@
   $BST2 = -tickpad 0.2 -xpad 3.5 -labelpadx 3.0 -ypad 0.5 -labelpady 0.0 +ypad 4.5 +labelpady 4.0 -labels 1001
   
-  section aM 0.00 0.66 0.65 0.33
-  section aR 0.00 0.33 0.65 0.33
-  section aD 0.00 0.00 0.65 0.33
-
-  section bM 0.65 0.66 0.35 0.33
-  section bR 0.65 0.33 0.35 0.33
-  section bD 0.65 0.00 0.35 0.33
+  section aM 0.00 0.66 0.65 0.33;  label -x mag -y "&sd&h mag"
+  section aR 0.00 0.33 0.65 0.33;  label -x mag -y "&sd&h R.A."
+  section aD 0.00 0.00 0.65 0.33;  label -x mag -y "&sd&h Dec" 
+
+  section bM 0.65 0.66 0.35 0.33;  label -x mag -y "&sd&h mag" 
+  section bR 0.65 0.33 0.35 0.33;  label -x mag -y "&sd&h R.A."
+  section bD 0.65 0.00 0.35 0.33;  label -x mag -y "&sd&h Dec" 
+
+  if (RA_PSF?[] == 0)
+    echo "no sources in datafile"
+    return
+  end
 
   set dR = 3600*(RA_PSF - Rref)*dcos(Dref)
@@ -256 +265 @@
  $Mdel = 0.25
  
+ if (Mref_out?[] == 0)
+   create magBin $Mmin $Mmax $Mdel
+   set numTotal = zero(magBin)
+   set numFound = zero(magBin)
+   set numPoss  = zero(magBin)
+   set numGood  = zero(magBin)
+   return
+ end
+
  delete -q  magBin numTotal numPoss numFound numGood
  for mag $Mmin $Mmax $Mdel

tags/ipp-ops-20220303-rc/ippTests/ippCheck/diffmask.pro

@@ -51 +51 @@
   # load the reference source table
   # how to choose the name of the filter?
-  data $refsrcs; 
+  data $refsrcs
+
+  # first check if the FITS table has any entries
+  read -fits DATA -sizes
+  if ($table:Ny == 0) 
+    echo "WARNING: empty reference table $refsrcs"
+
+    $OUT_FIELDS = 
+    foreach field $SMF_FIELDS_DIF $REF_FIELDS_DIF $XTR_FIELDS_DIF
+      $OUT_FIELDS = $OUT_FIELDS $field\_out
+      create -empty $field\_out
+    end
+    
+    sprintf output "%s/%s.%s.dif.stf" $OUTDIR $expname $OUTVERSION
+    write -fits DATA $output $OUT_FIELDS
+    return
+  end
+
   if ($USE_CAMSOURCE) 
    read -fits DATA Rref_out Dref_out Mref_out

tags/ipp-ops-20220303-rc/ippTests/ippCheck/ippCheck

@@ -59 +59 @@
   getargs -no-warp  BOOL  -var SKIP_WARP
   getargs -no-diff  BOOL  -var SKIP_DIFF
-  getargs -do-plots BOOL  -var DO_PLOTS;   
+  getargs -no-plots BOOL  -var SKIP_PLOTS;   
   getargs -use-refs BOOL  -var USE_REFSOURCE
   getargs -camera   VALUE -var CAMERA;
@@ -97 +97 @@
   end
 
-  if ($DO_PLOTS) plot.exposure.real $expname
+  if (not($SKIP_PLOTS)) plot.exposure.real $expname
 end
 

tags/ipp-ops-20220303-rc/ippTests/ippCheck/warpmask.pro

@@ -44 +44 @@
   # NOTE: Mref is chosen to match filter for this exposure
   data $refsrcs                 ;
+
+  # first check if the FITS table has any entries
+  read -fits DATA -sizes
+  if ($table:Ny == 0) 
+    echo "WARNING: empty reference table $refsrcs"
+
+    # generate empty output vectors for output file
+    $OUT_FIELDS = 
+    foreach field $REF_FIELDS_WRP $XTR_FIELDS_WRP
+      $OUT_FIELDS = $OUT_FIELDS $field\_out
+      create -empty $field\_out
+    end
+    
+    sprintf output "%s/%s.%s.wrp.stf" $OUTDIR $expname $OUTVERSION
+    write -fits DATA $output $OUT_FIELDS
+
+    return
+  end
+
   if ($USE_CAMSOURCE) 
    read -fits DATA Rref_out Dref_out Mref_out

tags/ipp-ops-20220303-rc/ippTests/ippCheck/warpstats.pro

@@ -24 +24 @@
    # results in magFound, fracFound, numFound
    calc.warpstats
-   if ($i == 0)
+   if (numTotal_sum?[] == 0)
      set numTotal_sum = numTotal
      set numPoss_sum  = numPoss
@@ -39 +39 @@
  end
    
- set numTotal = numTotal_sum
- set numPoss  = numPoss_sum
- delete numTotal_sum numPoss_sum
+ if (numTotal_sum?[] == 0)
+    # generate vectors for empty inputs
+   calc.warpstats
+ else   
+   set numTotal = numTotal_sum
+   set numPoss  = numPoss_sum
+   delete numTotal_sum numPoss_sum
+ end
 
  set fracPoss  = (numTotal > 0) ? numPoss  / numTotal : zero(numTotal)
@@ -89 +94 @@
  $Mdel = 0.25
  
+ if (Mref_out?[] == 0)
+   create magBin $Mmin $Mmax $Mdel
+   set numTotal = zero(magBin)
+   set numPoss  = zero(magBin)
+   return
+ end
+
  delete -q  magBin numTotal numPoss
  for mag $Mmin $Mmax $Mdel