Changeset 42056

Timestamp:

Feb 23, 2022, 3:57:05 PM (4 years ago)

Author:

eugene

Message:

a number of improvements to warp and diff

Location:

branches/eam_branches/ipp-20211108/ippTests/ippCheck

Files:

• ippCheck (modified) (2 diffs)
• smfcheck.pro (modified) (19 diffs)

branches/eam_branches/ipp-20211108/ippTests/ippCheck/ippCheck

@@ -224 +224 @@
 end
 
+macro init.stage
+  if ($0 < 2) 
+    echo "USAGE: init.stage (stage)"
+    break
+  end
+
+  if (("$1" == "cam") || ("$1" == "camera"))
+    init.camera
+    return
+  end
+  if (("$1" == "wrp") || ("$1" == "warp"))
+    init.warp
+    return
+  end
+  if (("$1" == "dif") || ("$1" == "diff"))
+    init.diff
+    return
+  end
+  
+  echo "ERROR : invalid stage $1"
+  break
+end
+
 macro show.usage
   echo "USAGE: ippCheck (mode) ..."
   echo "   EG: ippCheck mpc  (expname) (dgroup) [mpcfile]"
   echo "   EG: ippCheck dvo  (expname) (dgroup)"
-  echo "   EG: ippCheck smf  (outroot) (smf1) (smf2) [+plot]"
-  echo "   EG: ippCheck smf.dgrp.exp (outroot) (expname) (dgroup1) (dgroup2) [+plot]"
-  echo "   EG: ippCheck smf.dgrp.all (outroot) (dgroup1) (dgroup2) [+plot]"
+  echo "   EG: ippCheck smf (stage) (outroot) (smf1) (smf2) [+plot]"
+  echo "   EG: ippCheck cmf (stage) (outroot) (smf1) (smf2) [+plot]"
+  echo "   EG: ippCheck smf.dgrp.exp (stage) (outroot) (expname) (dgroup1) (dgroup2) [+plot]"
+  echo "   EG: ippCheck smf.dgrp.all (stage) (outroot) (dgroup1) (dgroup2) [+plot]"
   echo "   EG: ippCheck plot (expname)"
   echo "   EG: ippCheck plot.glob (expglob) (outroot)"
   echo "   EG: ippCheck mpcrefs (Tmin) (Tmax) (Rmin) (Rmax) (Dmin) (Dmax)"
+
+  echo " also equivalent:"
+  echo "   EG: ippCheck smf.compare.exp.by.datagroup (stage) (outroot) (expname) (dgroup1) (dgroup2) [+plot]"
+  echo "   EG: ippCheck smf.compare.all.by.datagroup (stage) (outroot) (dgroup1) (dgroup2) [+plot]"
 end
 
@@ -259 +287 @@
   end
 
-  if ("$argv:0" == "smf")
+  # compare output products for a single exposure
+  if (("$argv:0" == "smf") || ("$argv:0" == "cmf"))
+    if ($argv:n < 2) 
+      echo "USAGE: ippCheck smf (stage) (outroot) (smf1) (smf2) [+plot]"
+      echo "USAGE: ippCheck cmf (stage) (outroot) (smf1) (smf2) [+plot]"
+      echo "   EG: ippCheck smf camera  (cctest) neb:///gpc1/ipc/o6370g0351o.2133651.smf neb:///gpc1/ipc/o6370g0351o.2497362.smf [+plot]"
+      echo "   EG: ippCheck cmf warp    (wwtest) neb:///gpc1/ipc/o6370g0351o.wrp.2109044.skycell.1278.000.cmf neb:///gpc1/ipc/o6370g0351o.wrp.2465875.skycell.1278.000.cmf [+plot]"
+    end
+
+    init.stage $argv:1
+    list argv -join arglist -start 2
+
     smf.compare.all.extensions $arglist
     exit 0
   end
 
-  if ("$argv:0" == "smf.dgrp.exp")
-    smf.compare.exp.by.datagroups $arglist
-    exit 0
-  end
-
-  if ("$argv:0" == "smf.dgrp.all")
-    smf.compare.allexp.by.datagroups $arglist
-    exit 0
+  # compare output products for a single exposure by data_groups
+  if (("$argv:0" == "smf.compare.exp.by.datagroups") || ("$argv:0" == "smf.dgrp.exp"))
+    if ($argv:n < 2) 
+      echo "USAGE: ippCheck smf.dgrp.exp (stage) (outroot) (expname) (dgroup1) (dgroup2) [+plot]"
+      echo "   EG: ippCheck smf.dgrp.exp camera  (cctest) o6370g0351o MOPS.dailytestset.20190516ops40745  eamtest.20220208"
+      echo "   EG: ippCheck smf.dgrp.exp warp    (wwtest) o6370g0351o MOPS.dailytestset.20190516ops40745  eamtest.20220208"
+    end
+
+    init.stage $argv:1
+    list argv -join arglist -start 2
+
+    if (("$argv:1" == "cam") || ("$argv:1" == "camera"))
+      smf.compare.cam.exp.by.datagroups $arglist
+      exit 0
+    end
+    if (("$argv:1" == "wrp") || ("$argv:1" == "warp"))
+      smf.compare.wrp.exp.by.datagroups $arglist
+      exit 0
+    end
+    if (("$argv:1" == "dif") || ("$argv:1" == "diff"))
+      smf.compare.dif.exp.by.datagroups $arglist
+      exit 0
+    end
+  end
+
+  # compare all exposures for a pair of data_groups
+  if (("$argv:0" == "smf.compare.all.by.datagroups") || ("$argv:0" == "smf.dgrp.all"))
+    if ($argv:n < 2) 
+      echo "USAGE: ippCheck smf.dgrp.all (stage) (outroot) (dgroup1) (dgroup2) [+plot]"
+      echo "   EG: ippCheck smf.dgrp.all camera  (cctest) MOPS.dailytestset.20190516ops40745  eamtest.20220208"
+      echo "   EG: ippCheck smf.dgrp.all warp    (wwtest) MOPS.dailytestset.20190516ops40745  eamtest.20220208"
+    end
+
+    init.stage $argv:1
+    list argv -join arglist -start 2
+
+    if (("$argv:1" == "cam") || ("$argv:1" == "camera"))
+      smf.compare.cam.all.by.datagroups $arglist
+      exit 0
+    end
+    if (("$argv:1" == "wrp") || ("$argv:1" == "warp"))
+      smf.compare.wrp.all.by.datagroups $arglist
+      exit 0
+    end
+    if (("$argv:1" == "dif") || ("$argv:1" == "diff"))
+      smf.compare.dif.all.by.datagroups $arglist
+      exit 0
+    end
   end
 

branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro

@@ -1 +1 @@
 #!/usr/bin/env dvo
 # compare smf-to-smf
+# opihi verbose on
 
 if (not($?SAVEPLOT)) set SAVEPLOT = 0
+if (not($?SAVEDATA)) set SAVEDATA = 0
 
 module tap.pro
 
-macro smf.compare.allexp.by.datagroups
+###########
+macro smf.compare.cam.all.by.datagroups
   $CAMERA = gpc1  
 
   getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
   getargs -camera   VALUE -var CAMERA
 
   if ($0 != 4)
-    echo "USAGE: smf.compare.allexp.by.datagroups (outdir) (DG1) (DG2) [+plot] [-camera gpc2]"
+    echo "USAGE: smf.compare.cam.all.by.datagroups (outdir) (DG1) (DG2) [+plot] [-camera gpc2]"
     break
   end
@@ -28 +32 @@
   queue2book myqueue mybook
   
-  $camPath = NONE
   book npages mybook -var npages
   
@@ -37 +40 @@
   end
   
+  $options =
+  setargs +plot   BOOL  $SAVEPLOT
+  setargs +dump   BOOL  $SAVEDATA
+  setargs -camera VALUE $CAMERA
+
+  mkdir $outdir
+
   for i 0 $npages  
     sprint pagename page.%03d $i
     book getword mybook $pagename exp_name -var expname
-    $expnames:$i = $expname
-  end
-  $expnames:n = $npages
-
+
+    echo ippCheck smf.dgrp.exp camera $outdir/$expname.cam/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.cam.log
+    exec ippCheck smf.dgrp.exp camera $outdir/$expname.cam/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.cam.log
+  end
   book delete mybook
-
+end
+
+###########
+macro smf.compare.wrp.all.by.datagroups
+  $CAMERA = gpc1  
+
+  getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
+  getargs -camera   VALUE -var CAMERA
+
+  if ($0 != 4)
+    echo "USAGE: smf.compare.wrp.all.by.datagroups (outdir) (DG1) (DG2) [+plot] [-camera gpc2]"
+    break
+  end
+
+  local outdir dgrp1 dgrp2 command i npages expname
+  $outdir = $1
+  $dgrp1 = $2
+  $dgrp2 = $3
+
+  # identify the list of exposures in the data groups
+  $command = warptool -dbname $CAMERA -listrun -data_group $dgrp2
+  
+  queueload  myqueue -x "$command"
+  queue2book myqueue mybook
+  
+  book npages mybook -var npages
+  
+  if ($npages < 1) 
+   echo "ERROR: missing exposures for $dgrp2 : $command"
+   book delete mybook
+   next
+  end
+  
   $options =
   setargs +plot   BOOL  $SAVEPLOT
+  setargs +dump   BOOL  $SAVEDATA
   setargs -camera VALUE $CAMERA
 
   mkdir $outdir
 
-  for i 0 $expnames:n
-    echo smf.compare.exp.by.datagroups $outdir/$expnames:$i $expnames:$i $dgrp1 $dgrp2 $options
-    exec ippCheck smf.dgrp.exp $outdir/$expnames:$i $expnames:$i $dgrp1 $dgrp2 $options >& $outdir/$expnames:$i.log
-  end
-end
-
-macro smf.compare.exp.by.datagroups
+  for i 0 $npages  
+    sprint pagename page.%03d $i
+    book getword mybook $pagename exp_name -var expname
+
+    echo ippCheck smf.dgrp.exp warp $outdir/$expname.wrp/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.wrp.log
+    exec ippCheck smf.dgrp.exp warp $outdir/$expname.wrp/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.wrp.log
+  end
+  book delete mybook
+end
+
+###########
+macro smf.compare.dif.all.by.datagroups
   $CAMERA = gpc1  
 
   getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
   getargs -camera   VALUE -var CAMERA
 
+  if ($0 != 4)
+    echo "USAGE: smf.compare.dif.all.by.datagroups (outdir) (DG1) (DG2) [+plot] [-camera gpc2]"
+    break
+  end
+
+  local outdir dgrp1 dgrp2 command i npages expname
+  $outdir = $1
+  $dgrp1 = $2
+  $dgrp2 = $3
+
+  # identify the list of exposures in the data groups
+  $command = difftool -dbname $CAMERA -listrun -data_group $dgrp2
+  
+  queueload  myqueue -x "$command"
+  queue2book myqueue mybook
+  
+  $camPath = NONE
+  book npages mybook -var npages
+  
+  if ($npages < 1) 
+   echo "ERROR: missing exposures for $dgrp2 : $command"
+   book delete mybook
+   next
+  end
+  
+  $options =
+  setargs +plot   BOOL  $SAVEPLOT
+  setargs +dump   BOOL  $SAVEDATA
+  setargs -camera VALUE $CAMERA
+
+  mkdir $outdir
+
+  # we are selecting exp_name_1 so we know the exposure is the positive image (not template)
+  for i 0 $npages  
+    sprint pagename page.%03d $i
+    book getword mybook $pagename exp_name_1 -var expname
+
+    echo ippCheck smf.dgrp.exp diff $outdir/$expname.dif/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.dif.log
+    exec ippCheck smf.dgrp.exp diff $outdir/$expname.dif/$expname $expname $dgrp1 $dgrp2 $options >& $outdir/$expname.dif.log
+  end
+  book delete mybook
+end
+
+###########
+macro smf.compare.cam.exp.by.datagroups
+  $CAMERA = gpc1  
+
+  getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
+  getargs -camera   VALUE -var CAMERA
+
   if ($0 != 5)
-    echo "USAGE: smf.compare.by.datagroups (output) (expname) (DG1) (DG2) [+plot] [-camera gpc2]"
+    echo "USAGE: smf.compare.cam.exp.by.datagroups (output) (expname) (DG1) (DG2) [+plot] [-camera gpc2]"
     break
   end
@@ -75 +176 @@
   $dgrp2   = $4
 
-  get.camera.metadata.by.datagroup $CAMERA $expname $dgrp1
-  $smffile = $camPath.smf     
-  $smf1    = `neb-locate -p $smffile`
-
-  get.camera.metadata.by.datagroup $CAMERA $expname $dgrp2
-  $smffile = $camPath.smf     
-  $smf2    = `neb-locate -p $smffile`
+  # output goes to $camPath
+  get.camera.metadata.by.datagroup $CAMERA $expname $dgrp1; $smfBase1 = $camPath
+  get.camera.metadata.by.datagroup $CAMERA $expname $dgrp2; $smfBase2 = $camPath
 
   $options =
   setargs +plot   BOOL  $SAVEPLOT
-  smf.compare.all.extensions $output $smf1 $smf2 $options
-end
-
+  setargs +dump   BOOL  $SAVEDATA
+  echo smf.compare.all.extensions $output $smfBase1.smf $smfBase2.smf $options
+       smf.compare.all.extensions $output $smfBase1.smf $smfBase2.smf $options
+end
+
+# XXX needs some error handling
+macro smf.compare.wrp.exp.by.datagroups
+  $CAMERA = gpc1  
+
+  getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
+  getargs -camera   VALUE -var CAMERA
+
+  if ($0 != 5)
+    echo "USAGE: smf.compare.wrp.exp.by.datagroups (output) (expname) (DG1) (DG2) [+plot] [-camera gpc2]"
+    break
+  end
+
+  local output expname dgrp1 dgrp2 saveplot npages1 npages2 cmfBase1 cmfBase2 i
+  $output  = $1
+  $expname = $2
+  $dgrp1   = $3
+  $dgrp2   = $4
+
+  # XXX ensure warpBook1 & warpBook2 are deleted / empty first
+  get.warp.metadata.by.datagroup warpBook1 $CAMERA $expname $dgrp1
+  get.warp.metadata.by.datagroup warpBook2 $CAMERA $expname $dgrp2
+
+  book npages warpBook1 -var npages1
+  if ($npages1 < 1) 
+    echo "ERROR: missing warps for exposure : $expname ($dgrp1)"
+    book delete warpBook1 
+    book delete warpBook2
+    next
+  end
+ 
+  book npages warpBook2 -var npages2
+  if ($npages1 != $npages2) 
+    echo "WARNING: inconsistent number of warps for exposure : $expname ($npages1 vs $npages2)"
+  end
+ 
+  for i 0 $npages1
+    book getpage warpBook1 $i -var mypage1
+    book getword warpBook1 $mypage1 skycell_id -var skycellID
+    book getword warpBook1 $mypage1 path_base  -var cmfBase1
+
+    # select the page with the same skycell_id as cmfFile1
+    # XXX if this page is not found, raise an error / warning and skip the rest
+    book getpage warpBook2 0  -key skycell_id $skycellID -var mypage2
+    book getword warpBook2 $mypage2 path_base  -var cmfBase2
+
+    $options =
+    setargs +plot   BOOL  $SAVEPLOT
+    setargs +dump   BOOL  $SAVEDATA
+    echo smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
+         smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
+ end
+
+ book delete warpBook1
+ book delete warpBook2
+end
+
+# XXX needs some error handling
+macro smf.compare.dif.exp.by.datagroups
+  $CAMERA = gpc1  
+
+  getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
+  getargs -camera   VALUE -var CAMERA
+
+  if ($0 != 5)
+    echo "USAGE: smf.compare.dif.exp.by.datagroups (output) (expname) (DG1) (DG2) [+plot] [-camera gpc2]"
+    break
+  end
+
+  local i output expname dgrp1 dgrp2 saveplot npages1 npages2 cmfBase1 cmfBase2 bothways
+  $output  = $1
+  $expname = $2
+  $dgrp1   = $3
+  $dgrp2   = $4
+
+  get.diff.metadata.by.datagroup diffBook1 $CAMERA $expname $dgrp1
+  get.diff.metadata.by.datagroup diffBook2 $CAMERA $expname $dgrp2
+
+  book npages diffBook1 -var npages1
+  if ($npages1 < 1) 
+    echo "ERROR: missing diffs for exposure : $expname ($dgrp1)"
+    book delete diffBook1 
+    book delete diffBook2
+    next
+  end
+ 
+  book npages diffBook2 -var npages2
+  if ($npages1 != $npages2) 
+    echo "WARNING: inconsistent number of diffs for exposure : $expname ($npages1 vs $npages2)"
+  end
+ 
+  for i 0 $npages1
+    book getpage diffBook1 $i -var mypage1
+    book getword diffBook1 $mypage1 skycell_id -var skycellID
+    book getword diffBook1 $mypage1 path_base  -var cmfBase1
+
+    # select the page with the same skycell_id as cmfFile1
+    # XXX if this page is not found, raise an error / warning and skip the rest
+    book getpage diffBook2 0  -key skycell_id $skycellID -var mypage2
+    book getword diffBook2 $mypage2 path_base  -var cmfBase2
+    book getword diffBook2 $mypage2 bothways   -var bothways
+
+    $options =
+    setargs +plot   BOOL  $SAVEPLOT
+    setargs +dump   BOOL  $SAVEDATA
+    echo smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
+         smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
+
+    if ("$bothways" == "T")
+      echo smf.compare.all.extensions $output.$skycellID.inv $cmfBase1.inv.cmf $cmfBase2.inv.cmf $options
+           smf.compare.all.extensions $output.$skycellID.inv $cmfBase1.inv.cmf $cmfBase2.inv.cmf $options
+    end
+ end
+
+ book delete diffBook1
+ book delete diffBook2
+end
+
+## this function can be used for camera smf or warp & diff cmf files
 macro smf.compare.all.extensions
   # defaults to 'false' if +plot is not provided
   getargs +plot     BOOL  -var SAVEPLOT
+  getargs +dump     BOOL  -var SAVEDATA
 
   if ($0 != 4)
     echo "USAGE: smf.compare.all.extensions (output) (smf1) (smf2)"
-    break
-  end
-
-  local outroot smf1 smf2 extname mydir
+    echo " NOTE: smf files may be nebulous keys"
+    break
+  end
+
+  local outroot smfFile1 smfFile2 extname mydir
   $outroot = $1
-  $smf1 = $2
-  $smf2 = $3
+  $smfFile1 = $2
+  $smfFile2 = $3
+
+  # NOTE: smf1 and smf2 cannot be local variables (names are used in convert.neb.name)
+  convert.neb.name $smfFile1 smf1
+  convert.neb.name $smfFile2 smf2
+
+  if ("$smf1" == "") 
+    echo "ERROR: missing smf/cmf file : $smfFile1"
+    next
+  end
+  if ("$smf2" == "") 
+    echo "ERROR: missing smf/cmf file : $smfFile2"
+    next
+  end
 
   dirname $outroot -var mydir
@@ -108 +342 @@
   # XXX can we calculate the number of tests based on the number of smf extensions?
   # plan 3125
-  plan 3065
-
-  # load psfField definitions
-  # XXX these should depend on which file is being compared (camera, warp, diff)
+  ## XXX if smf.compare.all.extensions is called by an external function, we probably need a different plan, 
+  ## or we need to call 'plan' at the outer level?
+  plan $NTESTS_CMF
+
+  # Load psfField definitions.  These variables are set by the
+  # (init.camera, init.warp, init.diff) macros defined in .ippcheckrc.
+  # These are called by init.stage
   smf.compare.parse.book $PSF_FIELDS  psfFields
   smf.compare.parse.book $PSF_KEYS    psfKeys
@@ -119 +356 @@
   # generate a list of just the extension names:
   list tablelines -x "ftable -list $smf1"
+  if ($tablelines:n == 0)
+    echo "ERROR failed to read FITS extensions in file $smf1"
+    next
+  end
   smf.compare.extract.extnames ext0 tablelines
 
   list tablelines -x "ftable -list $smf2"
+  if ($tablelines:n == 0)
+    echo "ERROR failed to read FITS extensions in file $smf1"
+    next
+  end
   smf.compare.extract.extnames ext1 tablelines
 
+  # generate matched lists of extensions $match1:N == $match2:N
   smf.compare.match.extlists ext0 ext1
 
@@ -130 +376 @@
   $NfailPsf = 0
 
-  # XXX how to do the next steps if the numbers do not match?
+  # compare the matched extensions $ext1:N == $ext2:($match2:N)
   for Next1 0 $ext0:n
     $Next2 = $match1:$Next1
@@ -223 +469 @@
   # NOTE : t0 and t1 are used to label the sets of vectors
   smf.load.fields $smf1 $ext1 t1
+  $smf1_has_data = ($tfields:n > 0) && ($table:Ny > 0)
+
   smf.load.fields $smf2 $ext2 t2
+  $smf2_has_data = ($tfields:n > 0) && ($table:Ny > 0)
+
+  ok ($smf1_has_data == $smf2_has_data) "tables both have (or both lack) data"
+  if (not($smf1_has_data) || not($smf2_has_data))
+    echo "no data in tables $smf1 $ext1 or $smf2 $ext2, skipping"
+    # I should account for the number of skipped tests
+    return
+  end
 
   # generate the matched lists (index1, index2)
@@ -246 +502 @@
 
   data $filename
-  read -fits $extnum -extnum -list-fields -q; list tfields -join allfields
+  read -fits $extnum -extnum -list-fields -q
+  if ($tfields:n == 0) 
+    echo "FITS table in $filename is empty" 
+    return
+  end
+  if ($table:Ny == 0) 
+    echo "FITS table in $filename is empty" 
+    return
+  end
+  list tfields -join allfields
   read -fits $extnum -extnum $allfields
   
@@ -297 +562 @@
 
   # add a test here?
-  vstat -q dX
-  ok ($SIGMA < 0.1) "X position stdev $SIGMA < 0.1"
-  vstat -q dY
-  ok ($SIGMA < 0.1) "Y position stdev $SIGMA < 0.1"
+  # XXX this test is equivalent to tests of the fields below
+  # but cannot be as finely controlled
+  # vstat -q dX
+  # ok ($SIGMA < 0.1) "X position stdev $SIGMA < 0.1"
+  # vstat -q dY
+  # ok ($SIGMA < 0.1) "Y position stdev $SIGMA < 0.1"
 end
 
@@ -323 +590 @@
   list fields -split $allfields
   for i 0 $fields:n
-    smf.compare.onefield   $output $fields:$i $v1 $v2
+    smf.compare.onefield    $output $fields:$i $v1 $v2
     if (not($PASS)) set $NfailFieldsPSF = $NfailFieldsPSF + 1
     smf.fields.magplot.show $output $fields:$i $v1 $v2
+    smf.fields.match.dump   $output $fields:$i $v1 $v2
   end  
 
@@ -344 +612 @@
 end
 
+macro smf.fields.match.dump
+  if ($0 != 5)
+    echo "USAGE: smf.fields.match.dump (output) (field) (v1) (v2)"
+    echo " NOTE: run smf.pos.compare (or equiv) first to generate index1, index2"
+    break
+  end
+  
+  local output field v1 v2
+  $output = $1
+  $field  = $2
+  $v1     = $3
+  $v2     = $4
+
+  if (not($SAVEDATA)) return
+  write $output.$field.dat PSF_INST_MAG_$v1 PSF_INST_MAG_SIG_$v1 $field\_$v1 tmp_$v2 
+end
+
 macro smf.fields.magplot.show
   if ($0 != 5)
-    echo "USAGE: smf.fields.magplot.show (output) {field) (v1) (v2)"
+    echo "USAGE: smf.fields.magplot.show (output) (field) (v1) (v2)"
     echo " NOTE: run smf.pos.compare (or equiv) first to generate index1, index2"
     break
@@ -404 +689 @@
   # echo $field $useStat
 
+  # this variable tells the name of the vector which 
+  # should be used to select the values to test 
+  # the choice of minMag / maxMag depends on this value
+  $keepFieldName = PSF_INST_MAG_SIG
+
   delete -q dtmp
   if ("$useStat" == "VALUE")
     set dtmp = $statScale*(tmp_$v2 - $field\_$v1)
-    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+    subset dtmpS = dtmp where (($keepFieldName\_$v1 > $minMag) && ($keepFieldName\_$v1 < $maxMag))
   end
   if ("$useStat" == "IVALUE")
     set dtmp = tmp_$v2 - $field\_$v1
-    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag) && (tmp_$v2 != -4096) && ($field\_$v1 != -4096))
+    subset dtmpS = dtmp where (($keepFieldName\_$v1 > $minMag) && ($keepFieldName\_$v1 < $maxMag) && (tmp_$v2 != -4096) && ($field\_$v1 != -4096))
   end
   if ("$useStat" == "RATIO")
     set dtmp = $statScale(tmp_$v2 / $field\_$v1)
-    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+    subset dtmpS = dtmp where (($keepFieldName\_$v1 > $minMag) && ($keepFieldName\_$v1 < $maxMag))
   end
   if ("$useStat" == "RVALUE")
     set dtmp = $statScale*(tmp_$v2 - $field\_$v1) / $field\_$v1
-    subset dtmpS = dtmp where ((PSF_INST_MAG_$v1 > $minMag) && (PSF_INST_MAG_$v1 < $maxMag))
+    subset dtmpS = dtmp where (($keepFieldName\_$v1 > $minMag) && ($keepFieldName\_$v1 < $maxMag))
   end
   if (("$useStat" == "NONE") || ("$useStat" == "NULL"))
@@ -522 +812 @@
   # LAYOUT   KEY   KEY   KEY
   # PAGENAME VALUE VALUE VALUE
+
+  file $filename isFound
+  if (not($isFound)) 
+    echo "missing ippCheck config file $filename"
+    exit 2
+  end
 
   # NOTE: this list parsing strips out extra spaces and empty lines
@@ -633 +929 @@
   if ($range == 0.0)
     # if all values are the same, generate a vector with a single value of all points
-    create $3 $hist:median {$hist:median + 1.5}
-    set $2 = zero($3) + 0.0
-    $2[0] = _tmp_sort[]
+    delete -q $3
+    concat $hist:median $3
+    set $2 = zero($3) + _tmp_sort[]
     return    
   end
@@ -642 +938 @@
     # if all values are the same, generate a vector with a single value of all points
     create $3 0 1
-    set $2 = zero($3) + 0.0
-    $2[0] = _tmp_sort[]
+    set $2 = zero($3) + _tmp_sort[]
     return    
   end
@@ -660 +955 @@
  end
 
- local dbname expname camlabel
+ local dbname expname camdgrp
  $dbname   = $1
  $expname  = $2
  $camdgrp  = $3
 
-# need to generically handle the question of which version / label
-# this will be a problem for old exposures without a useful label (cleanup)
-# instead of label, we could specify the cam_id (which we might know)
-
+ book init mybook
+
+ # the camera data_group should select a unique exposure
  $command = camtool -dbname $dbname -processedexp -exp_name $expname -data_group $camdgrp
  
@@ -694 +988 @@
  book getword mybook $pagename cam_id    -var camID
  book delete mybook
+end
+
+################
+macro get.warp.metadata.by.datagroup
+ if ($0 != 5)
+  echo "USAGE: get.warp.metadata.by.datagroup (bookname) (camera) (expname) (data_group)"
+  echo "EX:    get.warp.metadata.by.datagroup warpBook gpc1 o9322g0253o MOPS.dailytestset.20190516ops40745"
+  echo "  sets warp book"
+  break
+ end
+
+ local warpbook dbname expname warpdgrp
+ $warpbook = $1
+ $dbname   = $2
+ $expname  = $3
+ $warpdgrp = $4
+
+ book init $warpbook
+
+ # the warp data_group should select a unique exposure
+ $command = warptool -dbname $dbname -warped -exp_name $expname -data_group $warpdgrp
+ 
+ queueload  myqueue -x "$command"
+ queue2book myqueue $warpbook -key skycell_id
+end
+
+################
+macro get.diff.metadata.by.datagroup
+ if ($0 != 5)
+  echo "USAGE: get.diff.metadata.by.datagroup (bookname) (camera) (expname) (data_group)"
+  echo "EX:    get.diff.metadata.by.datagroup diffBook gpc1 o9322g0253o MOPS.dailytestset.20190516ops40745"
+  echo "  sets diff book"
+  break
+ end
+
+ local diffbook dbname expname diffdgrp length
+ $diffbook = $1
+ $dbname   = $2
+ $expname  = $3
+ $diffdgrp = $4
+
+ book init $diffbook
+
+ # get the complete list of skyfiles for this exposure
+ # the diff data_group should select a unique exposure
+ # however, we do not know if the exposure was used as the positive or negative image:
+ $command = difftool -dbname $dbname -diffskyfile -exp_name $expname -data_group $diffdgrp 
+ queueload myqueue -x "$command"
+
+ queuesize myqueue -var length
+ if ($length == 0)
+   $command = difftool -dbname $dbname -diffskyfile -exp_name $expname -data_group $diffdgrp -template
+   queueload myqueue -x "$command"
+ end
+
+ queue2book myqueue $diffbook -key skycell_id
 end
 
@@ -778 +1128 @@
 end
 
+macro convert.neb.name
+  if ($0 != 3)
+    echo "USAGE: convert.neb.name (inname) (outname)"
+    echo " replaces outname with either real filename for inname (from nebulous if appropriate)"
+    break
+  end
+
+  local inname outname prefix
+  $inname = $1
+  $outname = $2
+
+  # is this a nebulous key?
+  substr $inname 0 5 prefix
+  if ("$prefix" == "neb:/")
+    $$outname = `neb-locate -p $inname`
+    if ("$$outname" == "") 
+      echo "ERROR: no nebulous entry for file : $inname"
+      next
+    end
+  else 
+    $$outname = $inname
+  end
+end
+
 ########## EXAMPLES: