Changeset 42058

Timestamp:

Feb 24, 2022, 2:36:09 PM (4 years ago)

Author:

eugene

Message:

add tap.skip, fix counting of tap tests based on context (warp, diff, camera), add option for ippCheck class to modify success criteria

Location:

branches/eam_branches/ipp-20211108/ippTests/ippCheck

Files:

• ippCheck (modified) (5 diffs)
• smfcheck.pro (modified) (22 diffs)
• tap.pro (modified) (3 diffs)

branches/eam_branches/ipp-20211108/ippTests/ippCheck/ippCheck

@@ -224 +224 @@
 end
 
+# The ippCheck class specifies the group of validation table to use for the analysis.
+# When comparing a specific processing, planned or expected deviations from previous 
+# results can be encoded uniquely in the class
+macro init.class
+
+  if (not($?IPPCHECK_CLASS))
+    $IPPCHECK_CLASS = default
+  end
+
+  # look for command-line argument of the form -class ()
+  local i
+
+# for i 0 $argv:n
+# NOTE: the opihi for-loop construct evaluates the loop conditions
+# only once before the loop starts.  Changing the value of argv:n
+# during the loop will cause problems in that context.  Also, we 
+# need to avoid incrementing $i when we strip off arguments
+
+# the opihi 'while' loop correctly reevaluates the condition on each loop
+
+  $i = 0
+  while ($i < $argv:n)
+    if ("$argv:$i" == "-class")
+      if ($argv:n < $i + 2)
+        echo "ERROR: incorrect usage: -class is missing classname: -class (classname)"
+        break
+      end
+      list argv -del -class
+      $IPPCHECK_CLASS = $argv:$i
+      list argv -del $IPPCHECK_CLASS
+    else
+      $i ++
+    end
+  end
+
+  # class-specific macros to define the class tables 
+  # should be listed in ~/.ippcheckrc 
+  ippcheck.class.$IPPCHECK_CLASS
+end
+
 macro init.stage
   if ($0 < 2) 
@@ -230 +270 @@
   end
 
-  if (("$1" == "cam") || ("$1" == "camera"))
-    init.camera
-    return
-  end
-  if (("$1" == "wrp") || ("$1" == "warp"))
-    init.warp
-    return
-  end
-  if (("$1" == "dif") || ("$1" == "diff"))
-    init.diff
-    return
-  end
+  if (("$1" == "cam") || ("$1" == "camera")) init.camera
+  if (("$1" == "wrp") || ("$1" == "warp"))   init.warp
+  if (("$1" == "dif") || ("$1" == "diff"))   init.diff
   
-  echo "ERROR : invalid stage $1"
-  break
+  if (not($?PSF_FIELDS))
+    echo "ERROR : invalid stage $1 or PSF_FIELDS not defined"
+    break
+  end
+  if (not($?PSF_KEYS))
+    echo "ERROR : PSF_KEYS not defined"
+    break
+  end
+  if (not($?HDR_KEYS))
+    echo "ERROR : HDR_KEYS not defined"
+    break
+  end
+
+  # Load PASS/FAIL criteria definitions
+  smf.compare.parse.book $PSF_FIELDS  psfFields
+  smf.compare.parse.book $PSF_KEYS    psfKeys
+  smf.compare.parse.book $HDR_KEYS    hdrKeys
+
+  # determine the number of tests per chip and cmf/smf file
+  local Nfields
+  book npages psfFields -var Nfields
+  $NTESTS_CHIP = $Nfields + 4
+  $NTESTS_CMF  = $NTESTS_CHIP * $NCHIPS + 2
 end
 
@@ -270 +322 @@
   end
 
+  # loads the requested class (finds and removes -class CLASSNAME) argument
+  init.class
+
+  # arguments to the script are given as $argv:0 - $argv:n
+  # argument lists are passed to macros called below with $arglist
   list argv -join arglist -start 1
 
@@ -294 +351 @@
       echo "   EG: ippCheck smf camera  (cctest) neb:///gpc1/ipc/o6370g0351o.2133651.smf neb:///gpc1/ipc/o6370g0351o.2497362.smf [+plot]"
       echo "   EG: ippCheck cmf warp    (wwtest) neb:///gpc1/ipc/o6370g0351o.wrp.2109044.skycell.1278.000.cmf neb:///gpc1/ipc/o6370g0351o.wrp.2465875.skycell.1278.000.cmf [+plot]"
+      exit 2
     end
 
@@ -299 +357 @@
     list argv -join arglist -start 2
 
+    # If we run on a single file, define the TAP test plan here.  The
+    # value of NTESTS_CHIP is set by the init.stage call based on psfFields
+    # The number of chips depends on the camera and stage
+    # For warps and diffs, the number of chips is 1 since a single cmf holds a single table
+    # there are 2 tests at the end of smf.compare.all.extensions
+    tap.plan $NTESTS_CMF
     smf.compare.all.extensions $arglist
+    tap.done
+
     exit 0
   end

branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro

@@ -180 +180 @@
   get.camera.metadata.by.datagroup $CAMERA $expname $dgrp2; $smfBase2 = $camPath
 
+  tap.plan $NTESTS_CMF
+
   $options =
   setargs +plot   BOOL  $SAVEPLOT
@@ -185 +187 @@
   echo smf.compare.all.extensions $output $smfBase1.smf $smfBase2.smf $options
        smf.compare.all.extensions $output $smfBase1.smf $smfBase2.smf $options
+  tap.done
 end
 
@@ -223 +226 @@
   end
  
+  tap.plan {$npages1 * $NTESTS_CMF}
+
   for i 0 $npages1
     book getpage warpBook1 $i -var mypage1
@@ -238 +243 @@
     echo smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
          smf.compare.all.extensions $output.$skycellID $cmfBase1.cmf $cmfBase2.cmf $options
- end
-
- book delete warpBook1
- book delete warpBook2
+  end
+  
+  book delete warpBook1
+  book delete warpBook2
+  
+  tap.done
 end
 
@@ -279 +286 @@
   end
  
+  tap.plan {2 * $npages1 * $NTESTS_CMF}
+
   for i 0 $npages1
     book getpage diffBook1 $i -var mypage1
@@ -299 +308 @@
       echo smf.compare.all.extensions $output.$skycellID.inv $cmfBase1.inv.cmf $cmfBase2.inv.cmf $options
            smf.compare.all.extensions $output.$skycellID.inv $cmfBase1.inv.cmf $cmfBase2.inv.cmf $options
+    else
+      tap.skip $NTESTS_CMF
     end
  end
@@ -329 +340 @@
   if ("$smf1" == "") 
     echo "ERROR: missing smf/cmf file : $smfFile1"
+    tap.skip $NTESTS_CMF
     next
   end
   if ("$smf2" == "") 
     echo "ERROR: missing smf/cmf file : $smfFile2"
+    tap.skip $NTESTS_CMF
     next
   end
@@ -338 +351 @@
   dirname $outroot -var mydir
   mkdir $mydir
-
-  # XXX this number depends on the schema of the smfs and the camera (gpc1 vs gpc2)
-  # XXX can we calculate the number of tests based on the number of smf extensions?
-  # plan 3125
-  ## XXX if smf.compare.all.extensions is called by an external function, we probably need a different plan, 
-  ## or we need to call 'plan' at the outer level?
-  plan $NTESTS_CMF
-
-  # Load psfField definitions.  These variables are set by the
-  # (init.camera, init.warp, init.diff) macros defined in .ippcheckrc.
-  # These are called by init.stage
-  smf.compare.parse.book $PSF_FIELDS  psfFields
-  smf.compare.parse.book $PSF_KEYS    psfKeys
-  smf.compare.parse.book $HDR_KEYS    hdrKeys
 
   # does the smf have the right number of extensions?
@@ -381 +380 @@
     if ($Next2 == -1) 
       echo "not ok : missing extenion $ext0:$Next1"
+      # XXX add a tap.skip here?
       next
     end
 
-    ok ($ext0:$Next1 == $ext1:$Next2) "matching ext names"
+    # tap.ok ($ext0:$Next1 == $ext1:$Next2) "matching ext names"
     $extname = $ext0:$Next1
     # NOTE: analysis below uses ext sequence number (i,j)
@@ -398 +398 @@
       echo "##### comparison results in $outroot.$extname.keys               #####"
       exec ippCheckKeys $smf1 $Next1 $smf2 $Next2 >& $outroot.$extname.keys
-      smf.compare.keys $outroot.$extname.keys hdrKeys
+      smf.compare.keys $outroot.$extname.keys hdrKeys; # TAP: 1
       if ($NfailKeys) set NfailHdr = $NfailHdr + 1
     end
@@ -408 +408 @@
       echo "##### comparison results in $outroot.$extname.keys               #####"
       exec ippCheckKeys $smf1 $Next1 $smf2 $Next2 >& $outroot.$extname.keys 
-      smf.compare.keys $outroot.$extname.keys psfKeys
+      smf.compare.keys $outroot.$extname.keys psfKeys; # TAP: 1
       if ($NfailKeys) set NfailPsf = $NfailPsf + 1
 
@@ -414 +414 @@
       echo "##### comparison results in $outroot.$extname.txt                  #####"
       smf.compare.onechip $outroot.$extname $smf1 {$Next1-1} $smf2 {$Next2-1}; # Note smf.compare.onechip counts from 0 = first extension
+      # TAP: 2 + Nfields
     end
 
@@ -422 +423 @@
       echo "##### comparison results in $outroot.$extname.keys               #####"
       exec ippCheckKeys $smf1 $Next1 $smf2 $Next2 >& $outroot.$extname.keys
-      smf.compare.keys $outroot.$extname.keys hdrKeys
+      smf.compare.keys $outroot.$extname.keys hdrKeys; # TAP: 1
       if ($NfailKeys) set NfailHdr = $NfailHdr + 1
     end
@@ -432 +433 @@
       echo "##### comparison results in $outroot.$extname.keys               #####"
       exec ippCheckKeys $smf1 $Next1 $smf2 $Next2 >& $outroot.$extname.keys 
-      smf.compare.keys $outroot.$extname.keys psfKeys
+      smf.compare.keys $outroot.$extname.keys psfKeys; # TAP: 1
       if ($NfailKeys) set NfailPsf = $NfailPsf + 1
 
@@ -440 +441 @@
     end
   end
-  ok ($NfailHdr == 0) "$NfailHdr HDR headers failed comparison" 
-  ok ($NfailPsf == 0) "$NfailPsf PSF headers failed comparison" 
-
-  done
+  tap.ok ($NfailHdr == 0) "$NfailHdr HDR headers failed comparison" ; # TAP: each file
+  tap.ok ($NfailPsf == 0) "$NfailPsf PSF headers failed comparison" ; # TAP: each file
 end
 
@@ -474 +473 @@
   $smf2_has_data = ($tfields:n > 0) && ($table:Ny > 0)
 
-  ok ($smf1_has_data == $smf2_has_data) "tables both have (or both lack) data"
+  tap.ok ($smf1_has_data == $smf2_has_data) "tables both have (or both lack) data" ; # TAP: each chip
   if (not($smf1_has_data) || not($smf2_has_data))
     echo "no data in tables $smf1 $ext1 or $smf2 $ext2, skipping"
     # I should account for the number of skipped tests
+    local Nfields
+    book npages psfFields -var Nfields
+    tap.skip {$Nfields + 1}
     return
   end
@@ -554 +556 @@
   # 
   output $output.txt
-  echo "$N1_keep & $N1_miss vs $N2_keep & $N2_miss"
+  echo "PSF FIELDS: $N1_keep & $N1_miss vs $N2_keep & $N2_miss"
   output stdout
-
-  # positional consistency:
-  set dX = X_PSF_$v1 - Xm_$v2
-  set dY = Y_PSF_$v1 - Ym_$v2
-
-  # add a test here?
-  # XXX this test is equivalent to tests of the fields below
-  # but cannot be as finely controlled
-  # vstat -q dX
-  # ok ($SIGMA < 0.1) "X position stdev $SIGMA < 0.1"
-  # vstat -q dY
-  # ok ($SIGMA < 0.1) "Y position stdev $SIGMA < 0.1"
 end
 
@@ -596 +586 @@
   end  
 
-  ok ($NfailFieldsPSF == 0) "$NfailFieldsPSF fields failed comparison"
-
+  tap.ok ($NfailFieldsPSF == 0) "$NfailFieldsPSF fields failed comparison"; # TAP: each chip
 end
 
@@ -715 +704 @@
     fprintf "%24s :  $PASS : SKIP" $field
     output stdout
+    tap.skip 1
     return
   end
@@ -728 +718 @@
   if ($hist:minVal < $minGood) set PASS = 0
   if ($hist:maxVal > $maxGood) set PASS = 0
+  if (isinf($hist:maxVal)) set PASS = 0
+  if (isinf($hist:minVal)) set PASS = 0
 
   # save the summary line to be written to a file by the calling function
@@ -734 +726 @@
   output stdout
 
-  ok ($PASS) "PSF table field $field comparison"
+  tap.ok ($PASS) "PSF table field $field comparison"; # TAP: each field
 end
 
@@ -796 +788 @@
     $NfailKeys ++
   end
-  ok ($NfailKeys == 0) "$NfailKeys keys failed comparison" 
+  tap.ok ($NfailKeys == 0) "$NfailKeys keys failed comparison" ; # TAP: called 2x per chip
 end
 

branches/eam_branches/ipp-20211108/ippTests/ippCheck/tap.pro

@@ -3 +3 @@
 if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 1
 
-macro plan
+macro tap.plan
  if ($0 != 2)
   echo "USAGE: plan (Ntest)"
@@ -12 +12 @@
  $tap_Npass = 0
  $tap_Nfail = 0
+ $tap_Nskip = 0
  $tap_Ntest = 0
 end
 
-macro done
- if (not($?tap_Nplan))
-  echo "ERROR: tap tests not planned"
+macro tap.skip
+ if ($0 != 2)
+  echo "USAGE skip (Nskip)"
   break
  end
 
- echo "planned $tap_Nplan tests, ran $tap_Ntest"
- echo "passed $tap_Npass tests, failed $tap_Nfail tests"
-
- if (($tap_Nfail == 0) && ($tap_Ntest == $tap_Nplan) && ($tap_Npass == $tap_Ntest))
-   echo "PASS: all tests run and passed"
-   return 1
- end
-
- if ($tap_Nfail > 0)
-   echo "FAIL: $tap_Nfail tests failed"
- end
-
- if ($tap_Ntest != $tap_Nplan)
-   echo "FAIL: not all tests run"
- end
-
- if ($tap_Ntest != $tap_Npass + $tap_Nfail)
-   echo "FAIL: inconsistent count: $tap_Ntest tests != $tap_Npass pass + $tap_Nfail fail"
- end
+  $tap_Nskip += $1
 end
 
-macro ok
+macro tap.ok
  if ($0 != 3)
-  echo "USAGE (condition) (label)"
+  echo "USAGE: ok (condition) (label)"
   break
  end
@@ -63 +46 @@
 end
 
+macro tap.done
+ if (not($?tap_Nplan))
+  echo "ERROR: tap tests not planned"
+  break
+ end
+
+ echo "planned $tap_Nplan tests, ran $tap_Ntest, skipped $tap_Nskip"
+ echo "passed $tap_Npass tests, failed $tap_Nfail tests"
+
+ if (($tap_Nfail == 0) && ($tap_Ntest + $tap_Nskip == $tap_Nplan) && ($tap_Npass == $tap_Ntest))
+   echo "PASS: all tests run and passed"
+   return 1
+ end
+
+ if ($tap_Nfail > 0)
+   echo "FAIL: $tap_Nfail tests failed"
+ end
+
+ if ($tap_Ntest + $tap_Nskip != $tap_Nplan)
+   echo "FAIL: not all tests run"
+ end
+
+ if ($tap_Ntest != $tap_Npass + $tap_Nfail)
+   echo "FAIL: inconsistent count: $tap_Ntest tests != $tap_Npass pass + $tap_Nfail fail"
+ end
+end
+