Index: /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro
===================================================================
--- /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro	(revision 42006)
+++ /branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro	(revision 42007)
@@ -1,2 +1,4 @@
+#!/usr/bin/env dvo
+
 # compare smf-to-smf
 if (not($?SAVEPLOT)) set SAVEPLOT = 0
@@ -13,5 +15,5 @@
 ### example of plotting all outputs for a single chip:
 macro go.sample
-  $SAVEPLOT = 0
+  $SAVEPLOT = 1
   $KAPA = kapa -noX
   smf.compare.parse.book fields.bk psfFields
@@ -19,17 +21,63 @@
 end
 
+### example of plotting all outputs for a single chip:
+macro go.warp
+  $SAVEPLOT = 1
+  $KAPA = kapa
+  smf.compare.parse.book fields.bk psfFields
+  smf.compare.onechip w1.phot.rnd w1.phot.v0.fpa.cmf w1.phot.v1.fpa.cmf 0
+end
+
+### example of plotting all outputs for a single chip:
+macro go.test
+  $SAVEPLOT = 1
+  $KAPA = kapa -noX
+#  smf.compare.parse.book test.bk psfFields
+#  smf.compare.onechip test o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
+  smf.compare.onechip o9451g0319o/o9451g0319o.XY01 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
+end
+
+### example of plotting all outputs for a single chip:
+macro full.test
+  if ($0 != 2)
+    echo "USAGE: full.test (version)"
+    break
+  end
+    
+  $SAVEPLOT = 0
+  $KAPA = kapa -noX
+  check.extensions o9451g0319o.$1/o9451g0319o.$1 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf
+end
+
 macro check.extensions
-  if ($0 != 3)
-    echo "USAGE: check.extensions (smf1) (smf2)"
-    break
-  end
-
-  plan 5
+  if ($0 != 4)
+    echo "USAGE: check.extensions (output) (smf1) (smf2)"
+    break
+  end
+
+  local outroot smf1 smf2 extname mydir
+  $outroot = $1
+  $smf1 = $2
+  $smf2 = $3
+
+  dirname $outroot -var mydir
+  mkdir $mydir
+
+  plan 3125
+
+  # load psfField definitions
+  smf.compare.parse.book fields.bk psfFields
+  smf.compare.parse.book hdrKeys.bk hdrKeys
+  smf.compare.parse.book psfKeys.bk psfKeys
 
   # 
-  list ext0 -x "ftable -list $1"
-  list ext1 -x "ftable -list $2"
+  list ext0 -x "ftable -list $smf1"
+  list ext1 -x "ftable -list $smf2"
 
   ok ($ext0:n == $ext1:n) "same number of extensions"
+
+  # track the number of chips for which the XYnn.hdr keys failed
+  $NfailHdr = 0
+  $NfailPsf = 0
 
   # XXX how to do the next steps if the numbers do not match?
@@ -40,22 +88,33 @@
     ok ($w0:0 == $w1:0) "matching ext names"
 
+    $extname = $w0:0
+
     # if extname is XYnn.hdr, compare the headers
-    strstr -q $w0:0 .hdr -start s1
+    strstr -q $extname .hdr -start s1
     if ($s1 == 4)
       $Next = $i - 1
-      echo $w0:0 == $Next
-      echo ippCheckKeys $1 $Next $2 $Next
-      exec ippCheckKeys $1 $Next $2 $Next
+      echo "# check keys for $extname == FITS entry number $Next"
+      exec ippCheckKeys $smf1 $Next $smf2 $Next >& $outroot.$extname.keys
+      smf.compare.keys $outroot.$extname.keys hdrKeys
+      if ($NfailKeys) set NfailHdr = $NfailHdr + 1
     end
 
     # if extname is XYnn.psf, compare the headers AND compare the table values
-    strstr -q $w0:0 .psf -start s1
+    strstr -q $extname .psf -start s1
     if ($s1 == 4)
       $Next = $i - 1
-      echo $w0:0 == $Next
-      echo ippCheckKeys $1 $Next $2 $Next
-      exec ippCheckKeys $1 $Next $2 $Next
-    end
-  end
+      echo "# check keys for $extname == FITS entry number $Next"
+      exec ippCheckKeys $smf1 $Next $smf2 $Next >& $outroot.$extname.keys 
+      smf.compare.keys $outroot.$extname.keys psfKeys
+      if ($NfailKeys) set NfailPsf = $NfailPsf + 1
+
+      echo "# check fields for $extname == FITS entry number $Next"
+      smf.compare.onechip $outroot.$extname $smf1 $smf2 {$Next - 1}; # Note smf.compare.onechip counts from 0 = first extension
+    end
+  end
+  ok ($NfailHdr == 0) "$NfailHdr HDR headers failed comparison" 
+  ok ($NfailPsf == 0) "$NfailPsf PSF headers failed comparison" 
+
+  done
 end
 
@@ -155,6 +214,9 @@
   set dY = Y_PSF_$v1 - Ym_$v2
 
-  vstat dX
-  vstat dY
+  # add a test here?
+  vstat -q dX
+  ok ($SIGMA < 0.1) "X position stdev < 0.1"
+  vstat -q dY
+  ok ($SIGMA < 0.1) "Y position stdev < 0.1"
 end
 
@@ -174,10 +236,16 @@
   exec rm -f $output.txt
 
+  $NfailFieldsPSF = 0
+
   # XXX this will fail if the two smfs do not have the same fields
   list fields -split $allfields
   for i 0 $fields:n
     smf.compare.onefield   $output $fields:$i $v1 $v2
-    smf.fields.magplot.foo $output $fields:$i $v1 $v2
+    if (not($PASS)) set $NfailFieldsPSF = $NfailFieldsPSF + 1
+    smf.fields.magplot.show $output $fields:$i $v1 $v2
   end  
+
+  ok ($NfailFieldsPSF == 0) "$NfailFieldsPSF fields failed comparison"
+
 end
 
@@ -192,7 +260,7 @@
 end
 
-macro smf.fields.magplot.foo
+macro smf.fields.magplot.show
   if ($0 != 5)
-    echo "USAGE: smf.fields.magplot.foo (output) {field) (v1) (v2)"
+    echo "USAGE: smf.fields.magplot.show (output) {field) (v1) (v2)"
     echo " NOTE: run smf.pos.compare (or equiv) first to generate index1, index2"
     break
@@ -224,5 +292,5 @@
 macro smf.compare.onefield
   if ($0 != 5)
-    echo "USAGE: smf.fields.onefield (output) {field) (v1) (v2)"
+    echo "USAGE: smf.fields.onefield (output) (field) (v1) (v2)"
     echo " NOTE: run smf.compare.position (or equiv) first to generate index1, index2"
     break
@@ -234,4 +302,6 @@
   $v1     = $3
   $v2     = $4
+
+  $PASS = 1; # if we hit any errors below, 
 
   # confirm that this field exists in the list of fields:
@@ -248,5 +318,5 @@
   book getword psfFields $field MIN_MAG  -var minMag
   book getword psfFields $field MAX_MAG  -var maxMag
-  echo $field $useStat
+  # echo $field $useStat
 
   delete -q dtmp
@@ -269,5 +339,5 @@
   if (("$useStat" == "NONE") || ("$useStat" == "NULL"))
     output $output.txt
-    fprintf "%24s :  1 : SKIP" $field
+    fprintf "%24s :  $PASS : SKIP" $field
     output stdout
     return
@@ -282,5 +352,4 @@
   vstat -q dtmpS
 
-  $PASS = 1
   if ($hist:minVal < $minGood) set PASS = 0
   if ($hist:maxVal > $maxGood) set PASS = 0
@@ -290,34 +359,43 @@
   fprintf "%24s : %2d : %12.6g %12.6g : %12.6g : %12.6g - %12.6g" $field $PASS $MEAN $SIGMA $hist:median $hist:minVal $hist:maxVal
   output stdout
-end
-
-macro smf.compare.parse.book
+
+  ok ($PASS) "PSF table field $field comparison"
+end
+
+macro smf.compare.keys
   if ($0 != 3)
-    echo "USAGE: smf.compare.parse.book (file) (book)"
-    break
-  end
-
-  local filename bookname pagename i j tmpline myChar
+    echo "USAGE: smf.compare.keys (file) (book)"
+    break
+  end
+
+  local filename bookname i tmpline newline Ns Ne keyword keymode myAction
   $filename = $1
   $bookname = $2
 
-  # the fields table has row-by-row pages with values:
-  # LAYOUT   KEY   KEY   KEY
-  # PAGENAME VALUE VALUE VALUE
-
+  # read in all lines of the file to be checked
   # NOTE: this list parsing strips out extra spaces and empty lines
   # ALSO: the code below handles quotes in the input file poorly
   list tmplines -x "cat $filename"
 
-  book init $bookname; # creates the book if it does not exist
-
-  $myWords:n = 0
+  $NfailKeys = 0
 
   for i 0 $tmplines:n
     $tmpline = $tmplines:$i
-    substr "$tmpline" 0 1 myChar
-    if ("$myChar" == "#") continue
-
-    list words -split $tmpline
+
+    # remove any trailing comments:
+    strstr -q "$tmpline" # -start Ns -length Ne
+
+    if ($Ns < 0)
+      # if no hash-mark is found, take the whole line
+      $newline = $tmpline
+    else
+      substr "$tmpline" 0 $Ns newline
+    end
+    if ("$newline" == "") continue; # if line contains only a comment, skip it
+
+    list words -split $newline
+
+    # keyword file should have lines of the form:
+    # MODE KEYWORD : VAL1 vs VAL2 : DFRAC
     if ($words:n < 2)
       echo "ERROR: no information on line $i" 
@@ -326,4 +404,69 @@
     end
 
+    $keymode = $words:0
+    $keyword = $words:1
+
+    # keywords where the MODE is MISS are always an error (missing keyword)
+    if ("$keymode" == "MISS")
+      echo "FAILURE: keyword $keyword MISSING"
+      $NfailKeys ++
+      continue
+    end
+
+    # what about BOOL & STR (mismatched values)
+
+    book getword $bookname $keyword ACTION -var myAction
+    if ("$myAction" == "IGNORE") continue
+    
+    echo "FAILURE: keyword $keyword differs between files"
+    $NfailKeys ++
+  end
+  ok ($NfailKeys == 0) "$NfailKeys keys failed comparison" 
+end
+
+macro smf.compare.parse.book
+  if ($0 != 3)
+    echo "USAGE: smf.compare.parse.book (file) (book)"
+    break
+  end
+
+  local filename bookname pagename i j tmpline myChar Ns Ne
+  $filename = $1
+  $bookname = $2
+
+  # the fields table has row-by-row pages with values:
+  # LAYOUT   KEY   KEY   KEY
+  # PAGENAME VALUE VALUE VALUE
+
+  # NOTE: this list parsing strips out extra spaces and empty lines
+  # ALSO: the code below handles quotes in the input file poorly
+  list tmplines -x "cat $filename"
+
+  book init $bookname; # creates the book if it does not exist
+
+  $myWords:n = 0
+
+  for i 0 $tmplines:n
+    $tmpline = $tmplines:$i
+
+    # remove any trailing comments:
+    strstr -q "$tmpline" # -start Ns -length Ne
+
+    if ($Ns < 0)
+      # if no hash-mark is found, take the whole line
+      $newline = $tmpline
+    else
+      substr "$tmpline" 0 $Ns newline
+    end
+    if ("$newline" == "") continue; # if line contains only a comment, skip it
+
+    list words -split $newline
+
+    if ($words:n < 2)
+      echo "ERROR: no information on line $i" 
+      echo "$tmpline"
+      break
+    end
+
     $pagename = $words:0
 
@@ -332,5 +475,5 @@
      if ("$pagename" != "LAYOUT")
        echo "ERROR: invalid page layout on first line" 
-       echo "$tmpline"
+       echo "$newline"
        break
      end
@@ -342,6 +485,6 @@
 
     if ($words:n != $myWords:n)
-       echo "ERROR: missing words for page on line $i" 
-       echo "$tmpline"
+       echo "ERROR: inconsistent number of words for page on line $i ($words:n vs $myWords:n)" 
+       echo "$newline"
        break
     end
@@ -425,45 +568,11 @@
 end
 
-# NOTE: in this method, match2d + reindex gives two pairs of vectors: the matched sources
-macro test.m1
-
-  $v1 = t0
-  $v2 = t1
-
-  match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v2 Y_PSF_$v2 $MATCH_RADIUS
-  reindex Xm_$v1 = X_PSF_$v1 using index1
-  reindex Ym_$v1 = Y_PSF_$v1 using index1
+if ($SCRIPT)
+  if ($argv:n != 1)
+    echo "USAGE: smfcheck.pro (version)"
+    break
+  end
   
-  reindex Xm_$v2 = X_PSF_$v2 using index2
-  reindex Ym_$v2 = Y_PSF_$v2 using index2
-
-  set dX = Xm_$v1 - Xm_$v2
-  set dY = Ym_$v1 - Ym_$v2
-
-  vstat dX
-  vstat dY
-end
-
-# NOTE: in this method, match2d -closest + reindex -keep-unmatched gives two pairs of vectors which match the original
-# source lists (unmatched are NAN)
-macro test.m2
-
-  $v1 = t0
-  $v2 = t1
-
-  match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v2 Y_PSF_$v2 $MATCH_RADIUS -closest
-  reindex Xm_$v1 = X_PSF_$v1 using index2 -keep-unmatched
-  reindex Ym_$v1 = Y_PSF_$v1 using index2 -keep-unmatched
-  set dX1 = Xm_$v1 - X_PSF_$v2
-  set dY1 = Ym_$v1 - Y_PSF_$v2
-  vstat dX1
-  vstat dY1
-  
-  reindex Xm_$v2 = X_PSF_$v2 using index1 -keep-unmatched
-  reindex Ym_$v2 = Y_PSF_$v2 using index1 -keep-unmatched
-  set dX2 = X_PSF_$v1 - Xm_$v2
-  set dY2 = Y_PSF_$v1 - Ym_$v2
-  vstat dX2
-  vstat dY2
-end
-
+  full.test $argv:0
+  exit 0
+end