IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42007


Ignore:
Timestamp:
Feb 1, 2022, 9:46:51 AM (4 years ago)
Author:
eugene
Message:

various improvements to key and field comparisons

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20211108/ippTests/ippCheck/smfcheck.pro

    r41993 r42007  
     1#!/usr/bin/env dvo
     2
    13# compare smf-to-smf
    24if (not($?SAVEPLOT)) set SAVEPLOT = 0
    … …  
    1315### example of plotting all outputs for a single chip:
    1416macro go.sample
    15   $SAVEPLOT = 0
     17  $SAVEPLOT = 1
    1618  $KAPA = kapa -noX
    1719  smf.compare.parse.book fields.bk psfFields
    … …  
    1921end
    2022
     23### example of plotting all outputs for a single chip:
     24macro go.warp
     25  $SAVEPLOT = 1
     26  $KAPA = kapa
     27  smf.compare.parse.book fields.bk psfFields
     28  smf.compare.onechip w1.phot.rnd w1.phot.v0.fpa.cmf w1.phot.v1.fpa.cmf 0
     29end
     30
     31### example of plotting all outputs for a single chip:
     32macro go.test
     33  $SAVEPLOT = 1
     34  $KAPA = kapa -noX
     35#  smf.compare.parse.book test.bk psfFields
     36#  smf.compare.onechip test o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
     37  smf.compare.onechip o9451g0319o/o9451g0319o.XY01 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf 2
     38end
     39
     40### example of plotting all outputs for a single chip:
     41macro full.test
     42  if ($0 != 2)
     43    echo "USAGE: full.test (version)"
     44    break
     45  end
     46   
     47  $SAVEPLOT = 0
     48  $KAPA = kapa -noX
     49  check.extensions o9451g0319o.$1/o9451g0319o.$1 o9451g0319o.1833521.cm.2467375.smf o9451g0319o.1833521.cm.2467377.smf
     50end
     51
    2152macro check.extensions
    22   if ($0 != 3)
    23     echo "USAGE: check.extensions (smf1) (smf2)"
    24     break
    25   end
    26 
    27   plan 5
     53  if ($0 != 4)
     54    echo "USAGE: check.extensions (output) (smf1) (smf2)"
     55    break
     56  end
     57
     58  local outroot smf1 smf2 extname mydir
     59  $outroot = $1
     60  $smf1 = $2
     61  $smf2 = $3
     62
     63  dirname $outroot -var mydir
     64  mkdir $mydir
     65
     66  plan 3125
     67
     68  # load psfField definitions
     69  smf.compare.parse.book fields.bk psfFields
     70  smf.compare.parse.book hdrKeys.bk hdrKeys
     71  smf.compare.parse.book psfKeys.bk psfKeys
    2872
    2973  #
    30   list ext0 -x "ftable -list $1"
    31   list ext1 -x "ftable -list $2"
     74  list ext0 -x "ftable -list $smf1"
     75  list ext1 -x "ftable -list $smf2"
    3276
    3377  ok ($ext0:n == $ext1:n) "same number of extensions"
     78
     79  # track the number of chips for which the XYnn.hdr keys failed
     80  $NfailHdr = 0
     81  $NfailPsf = 0
    3482
    3583  # XXX how to do the next steps if the numbers do not match?
    … …  
    4088    ok ($w0:0 == $w1:0) "matching ext names"
    4189
     90    $extname = $w0:0
     91
    4292    # if extname is XYnn.hdr, compare the headers
    43     strstr -q $w0:0 .hdr -start s1
     93    strstr -q $extname .hdr -start s1
    4494    if ($s1 == 4)
    4595      $Next = $i - 1
    46       echo $w0:0 == $Next
    47       echo ippCheckKeys $1 $Next $2 $Next
    48       exec ippCheckKeys $1 $Next $2 $Next
     96      echo "# check keys for $extname == FITS entry number $Next"
     97      exec ippCheckKeys $smf1 $Next $smf2 $Next >& $outroot.$extname.keys
     98      smf.compare.keys $outroot.$extname.keys hdrKeys
     99      if ($NfailKeys) set NfailHdr = $NfailHdr + 1
    49100    end
    50101
    51102    # if extname is XYnn.psf, compare the headers AND compare the table values
    52     strstr -q $w0:0 .psf -start s1
     103    strstr -q $extname .psf -start s1
    53104    if ($s1 == 4)
    54105      $Next = $i - 1
    55       echo $w0:0 == $Next
    56       echo ippCheckKeys $1 $Next $2 $Next
    57       exec ippCheckKeys $1 $Next $2 $Next
    58     end
    59   end
     106      echo "# check keys for $extname == FITS entry number $Next"
     107      exec ippCheckKeys $smf1 $Next $smf2 $Next >& $outroot.$extname.keys
     108      smf.compare.keys $outroot.$extname.keys psfKeys
     109      if ($NfailKeys) set NfailPsf = $NfailPsf + 1
     110
     111      echo "# check fields for $extname == FITS entry number $Next"
     112      smf.compare.onechip $outroot.$extname $smf1 $smf2 {$Next - 1}; # Note smf.compare.onechip counts from 0 = first extension
     113    end
     114  end
     115  ok ($NfailHdr == 0) "$NfailHdr HDR headers failed comparison"
     116  ok ($NfailPsf == 0) "$NfailPsf PSF headers failed comparison"
     117
     118  done
    60119end
    61120
    … …  
    155214  set dY = Y_PSF_$v1 - Ym_$v2
    156215
    157   vstat dX
    158   vstat dY
     216  # add a test here?
     217  vstat -q dX
     218  ok ($SIGMA < 0.1) "X position stdev < 0.1"
     219  vstat -q dY
     220  ok ($SIGMA < 0.1) "Y position stdev < 0.1"
    159221end
    160222
    … …  
    174236  exec rm -f $output.txt
    175237
     238  $NfailFieldsPSF = 0
     239
    176240  # XXX this will fail if the two smfs do not have the same fields
    177241  list fields -split $allfields
    178242  for i 0 $fields:n
    179243    smf.compare.onefield   $output $fields:$i $v1 $v2
    180     smf.fields.magplot.foo $output $fields:$i $v1 $v2
     244    if (not($PASS)) set $NfailFieldsPSF = $NfailFieldsPSF + 1
     245    smf.fields.magplot.show $output $fields:$i $v1 $v2
    181246  end 
     247
     248  ok ($NfailFieldsPSF == 0) "$NfailFieldsPSF fields failed comparison"
     249
    182250end
    183251
    … …  
    192260end
    193261
    194 macro smf.fields.magplot.foo
     262macro smf.fields.magplot.show
    195263  if ($0 != 5)
    196     echo "USAGE: smf.fields.magplot.foo (output) {field) (v1) (v2)"
     264    echo "USAGE: smf.fields.magplot.show (output) {field) (v1) (v2)"
    197265    echo " NOTE: run smf.pos.compare (or equiv) first to generate index1, index2"
    198266    break
    … …  
    224292macro smf.compare.onefield
    225293  if ($0 != 5)
    226     echo "USAGE: smf.fields.onefield (output) {field) (v1) (v2)"
     294    echo "USAGE: smf.fields.onefield (output) (field) (v1) (v2)"
    227295    echo " NOTE: run smf.compare.position (or equiv) first to generate index1, index2"
    228296    break
    … …  
    234302  $v1     = $3
    235303  $v2     = $4
     304
     305  $PASS = 1; # if we hit any errors below,
    236306
    237307  # confirm that this field exists in the list of fields:
    … …  
    248318  book getword psfFields $field MIN_MAG  -var minMag
    249319  book getword psfFields $field MAX_MAG  -var maxMag
    250   echo $field $useStat
     320  # echo $field $useStat
    251321
    252322  delete -q dtmp
    … …  
    269339  if (("$useStat" == "NONE") || ("$useStat" == "NULL"))
    270340    output $output.txt
    271     fprintf "%24s :  1 : SKIP" $field
     341    fprintf "%24s :  $PASS : SKIP" $field
    272342    output stdout
    273343    return
    … …  
    282352  vstat -q dtmpS
    283353
    284   $PASS = 1
    285354  if ($hist:minVal < $minGood) set PASS = 0
    286355  if ($hist:maxVal > $maxGood) set PASS = 0
    … …  
    290359  fprintf "%24s : %2d : %12.6g %12.6g : %12.6g : %12.6g - %12.6g" $field $PASS $MEAN $SIGMA $hist:median $hist:minVal $hist:maxVal
    291360  output stdout
    292 end
    293 
    294 macro smf.compare.parse.book
     361
     362  ok ($PASS) "PSF table field $field comparison"
     363end
     364
     365macro smf.compare.keys
    295366  if ($0 != 3)
    296     echo "USAGE: smf.compare.parse.book (file) (book)"
    297     break
    298   end
    299 
    300   local filename bookname pagename i j tmpline myChar
     367    echo "USAGE: smf.compare.keys (file) (book)"
     368    break
     369  end
     370
     371  local filename bookname i tmpline newline Ns Ne keyword keymode myAction
    301372  $filename = $1
    302373  $bookname = $2
    303374
    304   # the fields table has row-by-row pages with values:
    305   # LAYOUT   KEY   KEY   KEY
    306   # PAGENAME VALUE VALUE VALUE
    307 
     375  # read in all lines of the file to be checked
    308376  # NOTE: this list parsing strips out extra spaces and empty lines
    309377  # ALSO: the code below handles quotes in the input file poorly
    310378  list tmplines -x "cat $filename"
    311379
    312   book init $bookname; # creates the book if it does not exist
    313 
    314   $myWords:n = 0
     380  $NfailKeys = 0
    315381
    316382  for i 0 $tmplines:n
    317383    $tmpline = $tmplines:$i
    318     substr "$tmpline" 0 1 myChar
    319     if ("$myChar" == "#") continue
    320 
    321     list words -split $tmpline
     384
     385    # remove any trailing comments:
     386    strstr -q "$tmpline" # -start Ns -length Ne
     387
     388    if ($Ns < 0)
     389      # if no hash-mark is found, take the whole line
     390      $newline = $tmpline
     391    else
     392      substr "$tmpline" 0 $Ns newline
     393    end
     394    if ("$newline" == "") continue; # if line contains only a comment, skip it
     395
     396    list words -split $newline
     397
     398    # keyword file should have lines of the form:
     399    # MODE KEYWORD : VAL1 vs VAL2 : DFRAC
    322400    if ($words:n < 2)
    323401      echo "ERROR: no information on line $i"
    … …  
    326404    end
    327405
     406    $keymode = $words:0
     407    $keyword = $words:1
     408
     409    # keywords where the MODE is MISS are always an error (missing keyword)
     410    if ("$keymode" == "MISS")
     411      echo "FAILURE: keyword $keyword MISSING"
     412      $NfailKeys ++
     413      continue
     414    end
     415
     416    # what about BOOL & STR (mismatched values)
     417
     418    book getword $bookname $keyword ACTION -var myAction
     419    if ("$myAction" == "IGNORE") continue
     420   
     421    echo "FAILURE: keyword $keyword differs between files"
     422    $NfailKeys ++
     423  end
     424  ok ($NfailKeys == 0) "$NfailKeys keys failed comparison"
     425end
     426
     427macro smf.compare.parse.book
     428  if ($0 != 3)
     429    echo "USAGE: smf.compare.parse.book (file) (book)"
     430    break
     431  end
     432
     433  local filename bookname pagename i j tmpline myChar Ns Ne
     434  $filename = $1
     435  $bookname = $2
     436
     437  # the fields table has row-by-row pages with values:
     438  # LAYOUT   KEY   KEY   KEY
     439  # PAGENAME VALUE VALUE VALUE
     440
     441  # NOTE: this list parsing strips out extra spaces and empty lines
     442  # ALSO: the code below handles quotes in the input file poorly
     443  list tmplines -x "cat $filename"
     444
     445  book init $bookname; # creates the book if it does not exist
     446
     447  $myWords:n = 0
     448
     449  for i 0 $tmplines:n
     450    $tmpline = $tmplines:$i
     451
     452    # remove any trailing comments:
     453    strstr -q "$tmpline" # -start Ns -length Ne
     454
     455    if ($Ns < 0)
     456      # if no hash-mark is found, take the whole line
     457      $newline = $tmpline
     458    else
     459      substr "$tmpline" 0 $Ns newline
     460    end
     461    if ("$newline" == "") continue; # if line contains only a comment, skip it
     462
     463    list words -split $newline
     464
     465    if ($words:n < 2)
     466      echo "ERROR: no information on line $i"
     467      echo "$tmpline"
     468      break
     469    end
     470
    328471    $pagename = $words:0
    329472
    … …  
    332475     if ("$pagename" != "LAYOUT")
    333476       echo "ERROR: invalid page layout on first line"
    334        echo "$tmpline"
     477       echo "$newline"
    335478       break
    336479     end
    … …  
    342485
    343486    if ($words:n != $myWords:n)
    344        echo "ERROR: missing words for page on line $i"
    345        echo "$tmpline"
     487       echo "ERROR: inconsistent number of words for page on line $i ($words:n vs $myWords:n)"
     488       echo "$newline"
    346489       break
    347490    end
    … …  
    425568end
    426569
    427 # NOTE: in this method, match2d + reindex gives two pairs of vectors: the matched sources
    428 macro test.m1
    429 
    430   $v1 = t0
    431   $v2 = t1
    432 
    433   match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v2 Y_PSF_$v2 $MATCH_RADIUS
    434   reindex Xm_$v1 = X_PSF_$v1 using index1
    435   reindex Ym_$v1 = Y_PSF_$v1 using index1
     570if ($SCRIPT)
     571  if ($argv:n != 1)
     572    echo "USAGE: smfcheck.pro (version)"
     573    break
     574  end
    436575 
    437   reindex Xm_$v2 = X_PSF_$v2 using index2
    438   reindex Ym_$v2 = Y_PSF_$v2 using index2
    439 
    440   set dX = Xm_$v1 - Xm_$v2
    441   set dY = Ym_$v1 - Ym_$v2
    442 
    443   vstat dX
    444   vstat dY
    445 end
    446 
    447 # NOTE: in this method, match2d -closest + reindex -keep-unmatched gives two pairs of vectors which match the original
    448 # source lists (unmatched are NAN)
    449 macro test.m2
    450 
    451   $v1 = t0
    452   $v2 = t1
    453 
    454   match2d X_PSF_$v1 Y_PSF_$v1 X_PSF_$v2 Y_PSF_$v2 $MATCH_RADIUS -closest
    455   reindex Xm_$v1 = X_PSF_$v1 using index2 -keep-unmatched
    456   reindex Ym_$v1 = Y_PSF_$v1 using index2 -keep-unmatched
    457   set dX1 = Xm_$v1 - X_PSF_$v2
    458   set dY1 = Ym_$v1 - Y_PSF_$v2
    459   vstat dX1
    460   vstat dY1
    461  
    462   reindex Xm_$v2 = X_PSF_$v2 using index1 -keep-unmatched
    463   reindex Ym_$v2 = Y_PSF_$v2 using index1 -keep-unmatched
    464   set dX2 = X_PSF_$v1 - Xm_$v2
    465   set dY2 = Y_PSF_$v1 - Ym_$v2
    466   vstat dX2
    467   vstat dY2
    468 end
    469 
     576  full.test $argv:0
     577  exit 0
     578end
Note: See TracChangeset for help on using the changeset viewer.