Index: /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro
===================================================================
--- /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro	(revision 41992)
+++ /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro	(revision 41993)
@@ -6,4 +6,16 @@
 
 input tap.sh
+
+# for a single chip, I want outputs with names like:
+# expname.chipID.smfcmp.field.png
+# expname.chipID.smfcmp.txt
+
+### example of plotting all outputs for a single chip:
+macro go.sample
+  $SAVEPLOT = 0
+  $KAPA = kapa -noX
+  smf.compare.parse.book fields.bk psfFields
+  smf.compare.onechip o9451g0319o/o9451g0319o.XY01 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
+end
 
 macro check.extensions
@@ -48,16 +60,4 @@
 end
 
-
-# for a single chip, I want outputs with names like:
-# expname.chipID.smfcmp.field.png
-# expname.chipID.smfcmp.txt
-
-### example of plotting all outputs for a single chip:
-macro go.sample
-  $SAVEPLOT = 1
-  $KAPA = kapa -noX
-  smf.compare.onechip o9451g0319o/o9451g0319o.XY01 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
-end
-
 # generate plot and comparison txt file for one chip (extension)
 macro smf.compare.onechip
@@ -76,10 +76,17 @@
   mkdir $mydir
 
-  load.smf.fields $smf1 $extnum t0
-  load.smf.fields $smf2 $extnum t1
-
-  smf.pos.compare t0 t1
-  smf.compare.fields $output t0 t1
-  output stdout
+  # XXX TEMP: remove existing output file 
+  exec rm -f $output.txt
+
+  # load in the full set of data for this extension
+  # NOTE : t0 and t1 are used to label the sets of vectors
+  load.smf.fields $smf1 $extnum t1
+  load.smf.fields $smf2 $extnum t2
+
+  # generate the matched lists (index1, index2)
+  smf.compare.positions $output t1 t2
+
+  # compare the values in the matched pairs for each field
+  smf.compare.fields $output t1 t2
 end
 
@@ -108,7 +115,7 @@
 
 $MATCH_RADIUS = 1; # match radius in pixels
-macro smf.pos.compare
+macro smf.compare.positions
   if ($0 != 4)
-    echo "USAGE: smf.pos.compare (output) (v1) (v2)"
+    echo "USAGE: smf.compare.positions (output) (v1) (v2)"
     echo "generates matched position lists & stats"
     break
@@ -120,5 +127,5 @@
   $v2     = $3
 
-  match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v1 Y_PSF_$v1 $MATCH_RADIUS -closest
+  match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v2 Y_PSF_$v2 $MATCH_RADIUS -closest
 
   # initial questions to answer:
@@ -128,6 +135,6 @@
   reindex Ym_$v1 = Y_PSF_$v1 using index2 -keep-unmatched
 
-  reindex Xm_$v1 = X_PSF_$v1 using index1 -keep-unmatched
-  reindex Ym_$v1 = Y_PSF_$v1 using index1 -keep-unmatched
+  reindex Xm_$v2 = X_PSF_$v2 using index1 -keep-unmatched
+  reindex Ym_$v2 = Y_PSF_$v2 using index1 -keep-unmatched
  
   vstat -q Xm_$v1
@@ -135,14 +142,16 @@
   $N1_miss = Xm_$v1[] - $N1_keep
   
-  vstat -q Xm_$v1
+  vstat -q Xm_$v2
   $N2_keep = $NPTS
-  $N2_miss = Xm_$v1[] - $N2_keep
-  
+  $N2_miss = Xm_$v2[] - $N2_keep
  
+  # 
+  output $output.txt
   echo "$N1_keep & $N1_miss vs $N2_keep & $N2_miss"
+  output stdout
 
   # positional consistency:
-  set dX = X_PSF_$v1 - Xm_$v1
-  set dY = Y_PSF_$v1 - Ym_$v1
+  set dX = X_PSF_$v1 - Xm_$v2
+  set dY = Y_PSF_$v1 - Ym_$v2
 
   vstat dX
@@ -162,5 +171,5 @@
   $v2     = $3
 
-  smf.fields.magrange.init
+  smf.fields.magplot.init
   exec rm -f $output.txt
 
@@ -168,10 +177,10 @@
   list fields -split $allfields
   for i 0 $fields:n
-    $f = $fields:$i
-    smf.fields.magrange.plot $output $f $v1 $v2
+    smf.compare.onefield   $output $fields:$i $v1 $v2
+    smf.fields.magplot.foo $output $fields:$i $v1 $v2
   end  
 end
 
-macro smf.fields.magrange.init
+macro smf.fields.magplot.init
   clear -s
   antialias 0.4
@@ -183,7 +192,7 @@
 end
 
-macro smf.fields.magrange.plot
+macro smf.fields.magplot.foo
   if ($0 != 5)
-    echo "USAGE: smf.fields.magrange.plot (output) {field) (v1) (v2)"
+    echo "USAGE: smf.fields.magplot.foo (output) {field) (v1) (v2)"
     echo " NOTE: run smf.pos.compare (or equiv) first to generate index1, index2"
     break
@@ -196,22 +205,11 @@
   $v2     = $4
 
-  # plot the new values vs the old values (all old values)
-  reindex tmp_$v2 = $field\_$v2 using index1 -keep-unmatched
-  set dtmp = tmp_$v2 - $field\_$v1
-  hist.range dtmp Ntmp dx
-  vstat -q dtmp
-
-  # save the summary line to be written to a file by the calling function
-  output $output.txt
-  fprintf "%24s : %12.6g %12.6g : %12.6g : %12.6g - %12.6g" $field $MEAN $SIGMA $hist:median $hist:minVal $hist:maxVal
-  output stdout
-
-  if (isnan($hist:median))
-    continue
-  end
+  if (not(dtmp?[])) return; # this happens if the field is not listed in the fields table (or stat is NONE)
+  if (not($SAVEPLOT)) return
+  if (isnan($hist:median)) return
 
   clear
   section a
-  lim PSF_INST_MAG_$v1 $hist:minVal $hist:maxVal; box $BSTY; plot PSF_INST_MAG_$v1 dtmp -pt cir -op 0.3
+  lim PSF_INST_MAG_$v1 $hist:minPlt $hist:maxPlt; box $BSTY; plot PSF_INST_MAG_$v1 dtmp -pt cir -op 0.3
   section b
   if (dx[] == 1)
@@ -221,6 +219,138 @@
   end
  
-  if ($SAVEPLOT)
-    png -name $output.$field.png
+  png -name $output.$field.png
+end
+
+macro smf.compare.onefield
+  if ($0 != 5)
+    echo "USAGE: smf.fields.onefield (output) {field) (v1) (v2)"
+    echo " NOTE: run smf.compare.position (or equiv) first to generate index1, index2"
+    break
+  end
+  
+  local output field v1 v2
+  $output = $1
+  $field  = $2
+  $v1     = $3
+  $v2     = $4
+
+  # confirm that this field exists in the list of fields:
+  # XXX missing functions to test for book or page existence
+
+  # generate the comparison statistic
+  reindex tmp_$v2 = $field\_$v2 using index1 -keep-unmatched
+
+  # what statistic do we use to compare?
+  book getword psfFields $field STAT     -var useStat
+  book getword psfFields $field SCALE    -var statScale
+  book getword psfFields $field MIN_GOOD -var minGood
+  book getword psfFields $field MAX_GOOD -var maxGood
+  book getword psfFields $field MIN_MAG  -var minMag
+  book getword psfFields $field MAX_MAG  -var maxMag
+  echo $field $useStat
+
+  delete -q dtmp
+  if ("$useStat" == "VALUE")
+    set dtmp = $statScale*(tmp_$v2 - $field\_$v1)
+    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+  end
+  if ("$useStat" == "IVALUE")
+    set dtmp = tmp_$v2 - $field\_$v1
+    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag) && (tmp_$v2 != -4096) && ($field\_$v1 != -4096))
+  end
+  if ("$useStat" == "RATIO")
+    set dtmp = $statScale(tmp_$v2 / $field\_$v1)
+    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+  end
+  if ("$useStat" == "RVALUE")
+    set dtmp = $statScale*(tmp_$v2 - $field\_$v1) / $field\_$v1
+    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+  end
+  if (("$useStat" == "NONE") || ("$useStat" == "NULL"))
+    output $output.txt
+    fprintf "%24s :  1 : SKIP" $field
+    output stdout
+    return
+  end
+  if (dtmp?[] == 0)
+    echo "unknown STAT value $useStat"
+    break
+  end
+  
+  # measure the stats on the subset for the defined mag range
+  hist.range dtmpS Ntmp dx
+  vstat -q dtmpS
+
+  $PASS = 1
+  if ($hist:minVal < $minGood) set PASS = 0
+  if ($hist:maxVal > $maxGood) set PASS = 0
+
+  # save the summary line to be written to a file by the calling function
+  output $output.txt
+  fprintf "%24s : %2d : %12.6g %12.6g : %12.6g : %12.6g - %12.6g" $field $PASS $MEAN $SIGMA $hist:median $hist:minVal $hist:maxVal
+  output stdout
+end
+
+macro smf.compare.parse.book
+  if ($0 != 3)
+    echo "USAGE: smf.compare.parse.book (file) (book)"
+    break
+  end
+
+  local filename bookname pagename i j tmpline myChar
+  $filename = $1
+  $bookname = $2
+
+  # the fields table has row-by-row pages with values:
+  # LAYOUT   KEY   KEY   KEY
+  # PAGENAME VALUE VALUE VALUE
+
+  # NOTE: this list parsing strips out extra spaces and empty lines
+  # ALSO: the code below handles quotes in the input file poorly
+  list tmplines -x "cat $filename"
+
+  book init $bookname; # creates the book if it does not exist
+
+  $myWords:n = 0
+
+  for i 0 $tmplines:n
+    $tmpline = $tmplines:$i
+    substr "$tmpline" 0 1 myChar
+    if ("$myChar" == "#") continue
+
+    list words -split $tmpline
+    if ($words:n < 2)
+      echo "ERROR: no information on line $i" 
+      echo "$tmpline"
+      break
+    end
+
+    $pagename = $words:0
+
+    if ($myWords:n == 0)
+     # the first uncommented line should define the list of words
+     if ("$pagename" != "LAYOUT")
+       echo "ERROR: invalid page layout on first line" 
+       echo "$tmpline"
+       break
+     end
+
+     # save these words on the list of words
+     list myWords -copy words
+     next
+    end
+
+    if ($words:n != $myWords:n)
+       echo "ERROR: missing words for page on line $i" 
+       echo "$tmpline"
+       break
+    end
+
+    # the first word on the line gives the page name
+    book newpage $bookname $pagename
+
+    for j 1 $words:n
+      book setword $bookname $pagename $myWords:$j $words:$j
+    end
   end
 end
@@ -237,5 +367,19 @@
   local Nm Np No value10 value50 value90 range
 
-  set _tmp_sort = $1
+  $hist:minPlt = NAN
+  $hist:maxPlt = NAN
+  $hist:minVal = NAN
+  $hist:maxVal = NAN
+  $hist:delVal = NAN
+  $hist:median = NAN
+
+  if (not($1?[]))
+    echo "invalid vector $1"
+    break
+  end
+
+  subset _tmp_sort = $1 where not(isnan($1))
+  if (_tmp_sort[] < 2) return
+
   sort _tmp_sort
   $Nm = 0.1*_tmp_sort[]
@@ -248,8 +392,13 @@
   $value50 = _tmp_sort[$No]
 
-  # set the range to be 2x the 10 - 90 percentile range
+  # save the 10 and 90 percentile values
+  $hist:minVal = $value10
+  $hist:maxVal = $value90
+
+  # set the plotting range to be 3x the 10 - 90 percentile range
   $range = 3*($value90 - $value10)
-  $hist:minVal = $value50 - 0.5*$range
-  $hist:maxVal = $value50 + 0.5*$range
+  $hist:minPlt = $value50 - 0.5*$range
+  $hist:maxPlt = $value50 + 0.5*$range
+
   $hist:delVal = $range / 250
   $hist:median = $value50