IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38348


Ignore:
Timestamp:
Jun 2, 2015, 8:52:24 AM (11 years ago)
Author:
eugene
Message:

working on compression in dvo

Location:
branches/eam_branches/ohana.20150429/src
Files:
19 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20150429/src/addstar/test

  • branches/eam_branches/ohana.20150429/src/addstar/test/relphot.dvo

    r35760 r38348  
    11# -*-sh-*-
     2if (not($?VERBOSE)) set VERBOSE = 0
    23
    34input tap.dvo
    … …  
    4142  end
    4243
    43   echo relphot -D CATDIR catdir.test r -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0
    44   exec relphot -D CATDIR catdir.test r -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0 >& /dev/null
     44  $output = >& /dev/null
     45  if ($VERBOSE > 2)
     46    $output =
     47  end
     48  $command = relphot -D CATDIR catdir.test -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0 -images r
     49  echo $command
     50  exec $command $output
    4551
    4652  catdir catdir.test

  • branches/eam_branches/ohana.20150429/src/addstar/test/relphot.flatcorr.dvo

    r33653 r38348  
    1414# * no other mechanism to tell DVO about a flat-field correction
    1515
     16$TAP_BREAK = 1
     17$TAP_VERBOSE = 1
    1618input tap.dvo
    1719
    1820# set globals
    19 if (not($?PLOT)) set PLOT = 0
     21if (not($?PLOT)) set PLOT = 0
     22
     23$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets
     24$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact)
     25$DO_2MASS = 1
     26$DO_SYNTH = 1
     27$DO_TYCHO = 1
     28$NC_CLOUDS = 0.08
     29$UC_CLOUDS = 0.02
    2030
    2131# set various global variables
    2232macro init
     33  # generate a fake ptolemy.rc (to make this a contained test)
     34  exec rm -f ptolemy.rc
     35  output ptolemy.rc
     36    echo "\#"
     37    echo PHOTCODE_FILE dvo.photcodes
     38    echo GSCFILE GSCregions.tbl
     39    echo ZERO_PT 25.0
     40    echo ADDSTAR_RADIUS 1.0
     41    echo CATMODE                 SPLIT
     42    echo CATFORMAT               PS1_V5
     43    # echo CATFORMAT               PS1_V2 -- must be at least PS1_V3 for ubercaldist to work
     44    echo SKY_DEPTH               4
     45    echo CAMERA                  gpc1
     46    echo CAMERA_CONFIG           dvo.layout
     47    echo "input dvo.config"
     48  output stdout
     49
     50  # we need to define a photcode table consistent with the fake construction
     51  $zpt_nominal:g = 24.0
     52  $zpt_nominal:r = 24.5
     53
     54  # klam is negative, so klam*(airmass - 1) increases the zero point
     55  $klam_nominal:g = -0.15
     56  $klam_nominal:r = -0.10
     57
     58  # generate a fake dvo.photcode
     59  exec rm -f dvo.photcodes
     60  output dvo.photcodes
     61    echo "\# code  name                type    zero  slope offset c1    c2   slope   zero  equiv  sys scale   scale  sys     poor   bad     poor   bad"
     62    echo "1     g                    sec   0.000  0.000 0.000     1     3 0.0000     0  1051   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
     63    echo "2     r                    sec   0.000  0.000 0.000     2     3 0.0000     0  1052   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
     64    echo "10000 GPC1.g.XY00          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     65    echo "10001 GPC1.g.XY01          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     66    echo "10010 GPC1.g.XY10          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     67    echo "10011 GPC1.g.XY11          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     68    echo "10100 GPC1.r.XY00          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     69    echo "10101 GPC1.r.XY01          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     70    echo "10110 GPC1.r.XY10          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     71    echo "10111 GPC1.r.XY11          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
     72    echo "2011  2MASS_J              ref   0.000  0.000 0.000     -     - 0.0000     0     6   0.080 0.000 2.000  0.000   0x0000 0x0000  0x0000 0x0000"
     73    echo "3001  SYNTH.g              ref   0.000  0.000 0.000     -     - 0.0000     0     1     nan 0.000 2.000  0.600   0x0000 0x0000  0x0000 0x0000"
     74    echo "2020  TYCHO_B              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
     75  output stdout     
     76
    2377  $RA_CENTER = 10.0
    2478  $DEC_CENTER = 20.0
    … …  
    3286  $CHIP_DX = 1000
    3387  $CHIP_DY = 1000
    34 
    35   # images are loaded into dvo with GPC1 photcodes, so we need to get the nominal zps for those filters
    36   $zpt_nominal:g = 24.58
    37   $zpt_nominal:r = 24.80
    38   $zpt_nominal:i = 24.74
    39   $zpt_nominal:z = 24.26
    40   $zpt_nominal:y = 23.41
    41 
    42   # klam is negative, so klam*(airmass - 1) increase the zero point
    43   $klam_nominal:g = -0.15
    44   $klam_nominal:r = -0.10
    45   $klam_nominal:i = -0.04
    46   $klam_nominal:z = -0.03
    47   $klam_nominal:y = -0.03
     88  $DEFECT_FRAC = 0.05
     89  $OFFSET_FRAC_UC = 0.0
     90  $OFFSET_FRAC_NC = 0.5
    4891
    4992  # we have two sets of images: ubercaled and not-ubercaled
    5093
    51   # sequence to count images (only used in this function)
    52   create tmpseq 0 9
    53 
    54   $filt_uc:n = 9
    55   $filt_uc:0 = g
    56   $filt_uc:1 = g
    57   $filt_uc:2 = g
    58   $filt_uc:3 = r
    59   $filt_uc:4 = r
    60   $filt_uc:5 = r
    61   $filt_uc:6 = i
    62   $filt_uc:7 = i
    63   $filt_uc:8 = i
    64 
    65   # airmass slopes for these sequnece
     94  # the number of exposures is defined by filt_uc:n
     95  list filt_uc -split g g g g r r r r
     96  $filt_uc:n = 4
     97
     98  # tmpseq is used to generate vectors in this function
     99  create tmpseq 0 $filt_uc:n
     100
     101  # exptime and airmasses for uc images
     102  set exptime_uc = 10.0 + zero(tmpseq)
     103  set airmass_uc =  1.3 + zero(tmpseq)
    66104  set klam_uc = zero(tmpseq)
    67   for i 0 tmpseq[]
    68     klam_uc[$i] = $klam_nominal:$filt_uc:$i
    69   end   
    70 
    71   # mjd and zpt values for ubercal'ed imaged
    72   set exptime_uc = 10.0 + zero(tmpseq)
    73   set airmass_uc = 1.3 + zero(tmpseq)
     105  set zpt_uc = zero(tmpseq)
    74106
    75107  # ubercal zero points are defined as ZP_nominal + 2.5log(exptime) + K*(airmass - 1.0)
    76108  # note that K is defined as a negative value (is this sensible?)
    77   set zpt_uc = 25.0 + 2.5*log(exptime_uc) + klam_uc*(airmass_uc - 1.0) + tmpseq*0.0050 - 0.0025
    78 
    79   set mjd_uc = zero(zpt_uc)
    80   mjd_uc[0] = 55000.01
    81   mjd_uc[1] = 55000.02
    82   mjd_uc[2] = 55000.03
    83   mjd_uc[3] = 55015.01
    84   mjd_uc[4] = 55015.02
    85   mjd_uc[5] = 55015.03
    86   mjd_uc[6] = 55025.01
    87   mjd_uc[7] = 55025.02
    88   mjd_uc[8] = 55025.03
     109
     110  # zero points and airmass slopes for these sequnece
     111  for i 0 $filt_uc:n
     112    klam_uc[$i] = $klam_nominal:$filt_uc:$i
     113    zpt_uc[$i]  = $zpt_nominal:$filt_uc:$i + 2.5*log(exptime_uc[$i]) + klam_uc[$i]*(airmass_uc[$i] - 1.0) + $UC_CLOUDS*(rnd(klam_uc[$i]) - 0.5)
     114  end   
     115  set mjd_uc = 55000.00 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3)
    89116
    90117  # mjd and zpt values for not-ubercal'ed images
    91   $filt_nc:n = 9
    92   $filt_nc:0 = g
    93   $filt_nc:1 = g
    94   $filt_nc:2 = g
    95   $filt_nc:3 = r
    96   $filt_nc:4 = r
    97   $filt_nc:5 = r
    98   $filt_nc:6 = i
    99   $filt_nc:7 = i
    100   $filt_nc:8 = i
     118  list filt_nc -split g g g g r r r r
     119  $filt_nc:n = 4
     120
     121  # tmpseq is used to generate vectors in this function
     122  create tmpseq 0 $filt_nc:n
     123
     124  # place this within a valid season (55000.0 - 55010.0 - 55020.0 - 55030.0)
     125  set exptime_nc = 10.0 + zero(tmpseq)
     126  set airmass_nc =  1.6 + zero(tmpseq)
     127  set klam_nc = zero(tmpseq)
     128  set zpt_nc = zero(tmpseq)
    101129
    102130  # airmass slopes for these sequnece
    103   set klam_nc = zero(tmpseq)
    104   for i 0 tmpseq[]
    105     klam_nc[$i] = $klam_nominal:$filt_nc:$i
     131  for i 0 $filt_nc:n
     132    klam_nc[$i] = $klam_nominal:$filt_nc:$i
     133    zpt_nc[$i]  = $zpt_nominal:$filt_nc:$i + 2.5*log(exptime_nc[$i]) + klam_nc[$i]*(airmass_nc[$i] - 1.0) + $NC_CLOUDS*(rnd(klam_nc[$i]) - 0.5)
    106134  end   
    107 
    108   # place this within a valid season (55000.0 - 55010.0 - 55020.0 - 55030.0)
    109   set exptime_nc = 15.0 + zero(tmpseq)
    110   set airmass_nc = 1.6 + zero(tmpseq)
    111 
    112   # ubercal zero points are defined as ZP_nominal + 2.5log(exptime) + K*(airmass - 1.0)
    113   set zpt_nc = 25.0 + 2.5*log(exptime_nc) + klam_nc*(airmass_nc - 1.0) - tmpseq*0.0050 + 0.0025
    114 
    115   set mjd_nc = zero(zpt_nc)
    116   mjd_nc[0] = 55000.11
    117   mjd_nc[1] = 55000.12
    118   mjd_nc[2] = 55000.13
    119   mjd_nc[3] = 55015.11
    120   mjd_nc[4] = 55015.12
    121   mjd_nc[5] = 55015.13
    122   mjd_nc[6] = 55025.11
    123   mjd_nc[7] = 55025.12
    124   mjd_nc[8] = 55025.13
     135  set mjd_nc = 55000.10 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3)
     136
     137  delete tmpseq
    125138end
    126139
    … …  
    139152  mkdir $rootdir
    140153
     154  break -auto off
    141155  exec rm -rf $catdir
     156  exec rm -rf $fileroot*
     157  break -auto on
    142158
    143159  init
    144160  mkzptfile
    145   mkstars 500
     161  mkstars 1000
     162
     163  catdir $catdir
     164
     165  mk2mass $fileroot $catdir
     166  ck2mass addstar
     167 
     168  mksynth $fileroot $catdir
     169  cksynth addstar
     170
     171  # to add a valid tycho test, I need to generate a zpt table for tycho as well
     172  mktycho $fileroot $catdir
     173  cktycho addstar
    146174
    147175  tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3}
    … …  
    149177  # generate the basic images and check they were correctly ingested by dvo
    150178  mksequence $fileroot $catdir
    151   for i 0 mjd_uc[] 3
    152     ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw
    153   end
    154   for i 0 mjd_nc[] 3
    155     ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i raw
     179  for i 0 mjd_uc[]
     180    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw
     181  end
     182  for i 0 mjd_nc[]
     183    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i raw
    156184  end
    157185
    158186  # run setphot on the db and check that the images now match the expected values
    159   exec setphot -update -ubercal testzpt.fits -D CATDIR $catdir
    160   for i 0 mjd_uc[] 3
    161     ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i setphot_uc
    162   end
    163   for i 0 mjd_nc[] 3
    164     ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i setphot_nc
     187  tapEXEC setphot -update -ubercal testzpt.fits -D CATDIR $catdir
     188  for i 0 mjd_uc[]
     189    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i setphot_uc
     190  end
     191  for i 0 mjd_nc[]
     192    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i setphot_nc
    165193  end
    166194
    167195  # run relphot on the db and check that the images now match the expected values
    168   exec relphot g,r,i -v -region 9.5 10.5 19.5 20.5 -D CATDIR catdir.test -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -keep-ubercal -D IMAGE_OFFSET 0.5 -update >& tmp.log
    169   for i 0 mjd_uc[] 3
    170     ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot
    171   end
    172   for i 0 mjd_nc[] 3
    173     ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot
    174   end
     196  tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -D IMAGE_OFFSET 0.5 -update -synthphot_zpts $fileroot.synthzpts.fits
     197  for i 0 mjd_uc[]
     198    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
     199  end
     200  for i 0 mjd_nc[]
     201    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
     202  end
     203  ck2mass relphot
     204  cksynth relphot
     205  cktycho relphot
    175206
    176207  tapDONE
     208end
     209
     210macro try.relphot
     211  if ($0 != 3)
     212    echo "go (fileroot) (catdir)"
     213    break
     214  end
     215
     216  local i fileroot rootdir catdir
     217
     218  $fileroot = $1
     219  $catdir = $2
     220
     221  # run relphot on the db and check that the images now match the expected values
     222  tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -D IMAGE_OFFSET 0.5 -update -synthphot_zpts $fileroot.synthzpts.fits -nloop 24
     223
     224  for i 0 mjd_uc[]
     225    # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
     226      ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
     227  end
     228  for i 0 mjd_nc[]
     229    # ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
     230      ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
     231  end
    177232end
    178233
    … …  
    215270  $AIRMASS     = $5
    216271  $FILTER      = $6
    217   $MODE        = $7
     272  $FUNC_MODE   = $7
    218273
    219274  # XXX need a function to extract the nominal zpt for a given filter / photcode from the db
    … …  
    235290
    236291  skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2}
    237   mextract ra dec mag xccd yccd where (abs(time - $MJD_IMAGE) < 0.0001)
     292  mextract ra dec mag xccd yccd dbflags photflags where (abs(time - $MJD_IMAGE) < 0.0001)
    238293 
    239294  match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2
    … …  
    246301  set dm = mag - stars_mag_m
    247302
     303  set good_photom = (not(photflags & 0x02))
     304  foreach field dm xccd yccd
     305    subset $field\_good = $field if good_photom
     306  end
     307
    248308  if ($PLOT)
    249309    dev -n 1
    250     lim mag dm; clear; box; plot mag dm
    251   end
    252 
    253   set cell_xbin = int(xccd / ($CHIP_DX / $NCELL_X))
    254   set cell_ybin = int(yccd / ($CHIP_DY / $NCELL_Y))
    255   subset dm00 = dm if (cell_xbin == 0) && (cell_ybin == 0)
    256   subset dm01 = dm if (cell_xbin == 0) && (cell_ybin == 1)
    257   subset dm10 = dm if (cell_xbin == 1) && (cell_ybin == 0)
    258   subset dm11 = dm if (cell_xbin == 1) && (cell_ybin == 1)
     310    lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002)
     311    dev -n 2
     312    lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002)
     313    dev -n 3
     314    lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002)
     315    echo "red points are known bad photometry"
     316  end
     317
     318  # reproduce eddie's esoteric padding
     319  set cell_xbin = max(0 , min(1 , int((xccd_good +  8) / ($CHIP_DX / $NCELL_X))))
     320  set cell_ybin = max(0 , min(1 , int((yccd_good + 10) / ($CHIP_DY / $NCELL_Y))))
     321  subset dm00 = dm_good if (cell_xbin == 0) && (cell_ybin == 0)
     322  subset dm01 = dm_good if (cell_xbin == 0) && (cell_ybin == 1)
     323  subset dm10 = dm_good if (cell_xbin == 1) && (cell_ybin == 0)
     324  subset dm11 = dm_good if (cell_xbin == 1) && (cell_ybin == 1)
    259325
    260326  # uncorrected values behave like this:
    … …  
    273339  $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0)
    274340
    275   if ("$MODE" == "raw")
     341  if ("$FUNC_MODE" == "raw")
    276342    for ix 0 $NCELL_X
    277343      for iy 0 $NCELL_Y
    … …  
    280346 
    281347        set dm_adjust = dm$ix\$iy - $ZPT_NOMINAL + $ZPT_REAL_NORM + cell_off[$ix][$iy]
     348        # echo set dm_adjust = dm$ix\$iy - $ZPT_NOMINAL + $ZPT_REAL_NORM + cell_off[$ix][$iy]
     349
    282350        vstat -q dm_adjust
    283         tapOK {abs($MEAN) < 0.005} "addstar raw ZP $MJD_IMAGE $FILTER"
     351        sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : cell%d%d %6.3f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA $ix $iy cell_off[$ix][$iy] dm_adjust[]
     352        tapOK {abs($MEAN) < 0.005} "$line"
    284353      end
    285354    end
    … …  
    287356  end
    288357
    289   if ("$MODE" == "setphot_uc")
    290     vstat -q dm
    291     tapOK {abs($MEAN) < 0.005} "setphot_uc  ZP $MJD_IMAGE $FILTER"
     358  if ("$FUNC_MODE" == "setphot_uc")
     359    vstat -q dm_good
     360    sprintf line "setphot_uc  ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
     361    tapOK {abs($MEAN) < 0.005} "$line"
     362
     363    subset tmp = dm if (dbflags & 0x00008000)
     364    tapOK {tmp[] == dm[]} "ubercal detections have ubercal bit set"
     365
     366    imextract MCAL FLAGS where (abs(time - $MJD_IMAGE) < 0.0001)
     367    subset tmp = FLAGS if (FLAGS & 0x0000200)
     368    tapOK {tmp[] == FLAGS[]} "ubercal images have ubercal bit set"
     369   
    292370    return
    293371  end
    294372
    295   if ("$MODE" == "setphot_nc")
    296     vstat -q dm
    297     tapOK {abs($MEAN - $ZPT_NOMINAL + $ZPT_REAL_NORM) < 0.005} "setphot_nc  ZP $MJD_IMAGE $FILTER"
     373  if ("$FUNC_MODE" == "setphot_nc")
     374    vstat -q dm_good
     375    sprintf line "setphot_nc  ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
     376    tapOK {abs($MEAN - $ZPT_NOMINAL + $ZPT_REAL_NORM) < 0.005} "$line"
     377
     378    subset tmp = dm if (not(dbflags & 0x00008000))
     379    tapOK {tmp[] == dm[]} "non-ubercal detections do NOT have ubercal bit set"
     380
     381    imextract FLAGS where (abs(time - $MJD_IMAGE) < 0.0001)
     382    subset tmp = FLAGS if (not(FLAGS & 0x0000200))
     383    tapOK {tmp[] == FLAGS[]} "non-ubercal images do NOT have ubercal bit set"
     384   
    298385    return
    299386  end
    300387
    301   if ("$MODE" == "relphot")
    302     vstat -q dm
    303     tapOK {abs($MEAN) < 0.005} "relphot    ZP $MJD_IMAGE $FILTER"
     388  if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc"))
     389    vstat -q dm_good
     390    sprintf line "relphot     ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
     391
     392    if ("$FUNC_MODE" == "relphot_uc")
     393      tapOK {abs($MEAN) < 0.001} "$line"
     394      tapOK {abs($SIGMA) < 0.001} "$line"
     395    end
     396    if ("$FUNC_MODE" == "relphot_nc")
     397      # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias
     398      tapOK {abs($MEAN) < 0.020} "$line"
     399      tapOK {abs($SIGMA) < 0.020} "$line"
     400    end
     401
     402    mextract ra dec mag dbflags
     403    subset rs = ra  if ((dbflags & 0x8000) && (mag > 13.5))
     404    subset ds = dec if ((dbflags & 0x8000) && (mag > 13.5))
     405    avmatch rs ds 0.1 $FILTER\:flags $FILTER\:uc_dist
     406
     407    subset tmp = rs if (($FILTER\:flags & 0x8) || ($FILTER\:flags & 0x4))
     408    tapOK {tmp[] == rs[]} "objects with ubercal detections have ubercal bit set or used SYNTH photometry (tmp[] vs rs[])"
     409 
     410    subset tmp = rs if (($FILTER\:uc_dist < 0.1) || ($FILTER\:flags & 0x4))
     411    tapOK {tmp[] == rs[]} "objects with ubercal detections have ubercal dist of 0.0 or used SYNTH photometry"
    304412    return
    305413  end
     414
     415  echo "ERROR: unknown mode for ckexposure : $FUNC_MODE"
    306416
    307417end
    … …  
    316426
    317427  # we have defined a set of mjd and zero point values for ubercal'ed images
    318   # for i 0 mjd_uc[]
    319   for i 0 mjd_uc[] 3
     428  for i 0 mjd_uc[]
    320429    mkexposure $1.uc.$i $RA_CENTER $DEC_CENTER zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] mjd_uc[$i] $filt_uc:$i $2
    321430  end
    322431
    323432  # we have a second set of mjd and zero point values for non-ubercal'ed images
    324   # for i 0 mjd_nc[]
    325   for i 0 mjd_nc[] 3
     433  for i 0 mjd_nc[]
    326434    mkexposure $1.nc.$i $RA_CENTER $DEC_CENTER zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] mjd_nc[$i] $filt_nc:$i $2
    327435  end
    … …  
    334442 end
    335443
    336  local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR
     444 local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd
    337445
    338446 $ROOT    = $1
    … …  
    366474    $options = -mjd $MJD
    367475    $options = $options -radec $ra $dec
    368     $options = $options -type PS1_V2
     476
     477    #$options = $options -type PS1_V2
     478    #$options = $options -type PS1_V3
     479    #$options = $options -type PS1_V4
     480    $options = $options -type PS1_V5
     481
    369482    $options = $options -coords
    370483    $options = $options -photcode GPC1.$FILTER.XY$ix\$iy
    371484    $options = $options -no-noise
     485    $options = $options -flags
     486    $options = $options -bad-psfqf-frac $DEFECT_FRAC
    372487    $options = $options -size $CHIP_DX $CHIP_DY
    373488    $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY}
    374489    $options = $options -airmass $AIRMASS
    375490    $options = $options -exptime $EXPTIME
    376     exec mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options
    377     echo mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options
    378    
     491
     492    tapEXEC mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options
     493
    379494    # the fake images have inconsistent ra,dec and airmass,sidtime values
    380     echo addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass
    381     exec addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass >& tmp.log
     495    tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass
    382496  end
    383497 end
    … …  
    405519  subset stars_mr = stars_mag if valid
    406520
    407   set cell_x = int(stars_x / ($CHIP_DX / $NCELL_X))
    408   set cell_y = int(stars_y / ($CHIP_DX / $NCELL_X))
     521  # note eddie's esoteric padding rule:
     522  set cell_x = max(0 , min(1 , int((stars_x +  8) / ($CHIP_DX / $NCELL_X))))
     523  set cell_y = max(0 , min(1 , int((stars_y + 10) / ($CHIP_DX / $NCELL_X))))
    409524
    410525  # eddie says: M_real = M_inst + zpt + ff_offset
    … …  
    418533  end
    419534
    420   write -f "%10.6f %10.6f %7.3f  %6.1f %6.1f %7.3f" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi
     535  # create bad detections / detections to ignore or avoid, save to a file?
     536
     537  # create 3 types of things to ignore / avoid
     538  # * badPhot (deviate the photometry)
     539  # * badFlag (set a bad flag bit)
     540  # * poorFlag (set a bad flag bit)
     541  set badPhot  = rnd(stars_r) <  $DEFECT_FRAC
     542  set badFlag  = rnd(stars_r) <  $DEFECT_FRAC
     543  set poorFlag = rnd(stars_r) <  $DEFECT_FRAC
     544
     545  # create deviant photometry (1%)
     546  # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated.  This is a simple way to pass the info
     547  # note that outlier rejection only occurs for Nmeas > 5
     548  set stars_mi = stars_mi + 0.2*badPhot
     549  set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot
     550
     551  # bad flags that we sould respect:
     552  # PM_SOURCE_MODE_FAIL             = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero)
     553  # PM_SOURCE_MODE_SATSTAR          = 0x00000080, ///< Source model peak is above saturation
     554  # PM_SOURCE_MODE_BADPSF           = 0x00000400, ///< Failed to get good estimate of object's PSF
     555  # PM_SOURCE_MODE_DEFECT           = 0x00000800, ///< Source is thought to be a defect
     556  # PM_SOURCE_MODE_SATURATED        = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail)
     557  # PM_SOURCE_MODE_CR_LIMIT         = 0x00002000, ///< Source has crNsigma above limit
     558  # PM_SOURCE_MODE_MOMENTS_FAILURE  = 0x00008000, ///< could not measure the moments
     559  # PM_SOURCE_MODE_SKY_FAILURE      = 0x00010000, ///< could not measure the local sky
     560  # PM_SOURCE_MODE_SKYVAR_FAILURE   = 0x00020000, ///< could not measure the local sky variance
     561  # PM_SOURCE_MODE_SIZE_SKIPPED     = 0x10000000, ///< size could not be determined
     562
     563  # poor flags that we should respect:
     564  # PM_SOURCE_MODE_POOR             = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?)
     565  # PM_SOURCE_MODE_PAIR             = 0x00000020, ///< Source fitted with a double psf
     566  # PM_SOURCE_MODE_BLEND            = 0x00000100, ///< Source is a blend with other sources
     567  # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N
     568  # PM_SOURCE_MODE_BLEND_FIT        = 0x00400000, ///< source was fitted as a blend
     569  # PM_SOURCE_MODE_ON_SPIKE         = 0x20000000, ///< peak lands on diffraction spike
     570  # PM_SOURCE_MODE_ON_GHOST         = 0x40000000, ///< peak lands on ghost or glint
     571  # PM_SOURCE_MODE_OFF_CHIP         = 0x80000000, ///< peak lands off edge of chip
     572
     573  write -f "%10.6f %10.6f %7.3f  %6.1f %6.1f %7.3f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl
    421574end
    422575
    … …  
    439592  set stars_ra  = $RA_CENTER  + $RA_RANGE  * (rnd(tmp) - 0.5) / 3600 / dcos ($DEC_CENTER)
    440593  set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) / 3600
    441   set stars_mag = 15.0 + 4.0 * rnd(tmp)
     594  set stars_mag = 10.0 + 10.0 * rnd(tmp)
     595  set stars_dmag = 0.02 + 0.1*rnd(tmp)
     596end
     597
     598macro mk2mass
     599  if ($0 != 3)
     600    echo "USAGE: mk2mass (fileroot) (catdir)"
     601    break
     602  end
     603
     604  if (not($DO_2MASS)) return
     605
     606  local ROOT CATDIR
     607
     608  $ROOT    = $1
     609  $CATDIR  = $2
     610
     611  set stars_flags = 0x00c00000*(rnd(stars_ra) < 0.1) + 0x7*(rnd(stars_ra) < 0.8)
     612  write $ROOT.2mass.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" stars_ra stars_dec stars_mag stars_dmag stars_flags
     613
     614  tapEXEC addstar -ref $ROOT.2mass.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode 2MASS_J
     615
     616  set r_2mass = stars_ra
     617  set d_2mass = stars_dec
     618  set m_2mass = stars_mag
     619  set dm_2mass = stars_dmag
     620  set fl_2mass = stars_flags
     621end
     622
     623macro ck2mass
     624  if ($0 != 2)
     625    echo "USAGE: ck2mass (stage)"
     626    break
     627  end
     628
     629  if (not($DO_2MASS)) return
     630
     631  if ("$1" == "addstar")
     632    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2}
     633    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 2011)
     634    tapOK {ra[] == r_2mass[]} "read in as many 2mass stars as we generated"
     635   
     636    match2d -closest ra dec r_2mass d_2mass 0.001 -index1 index1 -index2 index2
     637    subset tmp = index1 if (index1 >= 0)
     638    tapOK {tmp[] == ra[]} "2mass sources match those generated"
     639  end
     640
     641  if ("$1" == "relphot")
     642    subset rs = r_2mass if (fl_2mass & 0x00c00000)
     643    subset ds = d_2mass if (fl_2mass & 0x00c00000)
     644    avmatch rs ds 0.1 flags
     645    subset tmp = rs if (flags & 0x02000000); # extended in 2mass
     646    tapOK {tmp[] == rs[]} "2mass extended objects properly marked"
     647   
     648    subset rs = r_2mass if (fl_2mass & 0x07)
     649    subset ds = d_2mass if (fl_2mass & 0x07)
     650    avmatch rs ds 0.1 flags
     651    subset tmp = rs if (flags & 0x08000000); # extended in 2mass
     652    tapOK {tmp[] == rs[]} "2mass good objects properly marked"
     653  end
     654end
     655
     656macro mksynth
     657  if ($0 != 3)
     658    echo "USAGE: mksynth (fileroot) (catdir)"
     659    break
     660  end
     661
     662  if (not($DO_SYNTH)) return
     663
     664  local ROOT CATDIR
     665
     666  $ROOT    = $1
     667  $CATDIR  = $2
     668 
     669  subset r_synth  = stars_ra    if (stars_mag < 17)
     670  subset d_synth  = stars_dec   if (stars_mag < 17)
     671  subset m_synth  = stars_mag   if (stars_mag < 17)
     672  subset dm_synth = stars_dmag  if (stars_mag < 17)
     673  subset fl_synth = stars_flags if (stars_mag < 17)
     674
     675  write $ROOT.synth.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" r_synth d_synth m_synth dm_synth fl_synth
     676
     677  tapEXEC addstar -ref $ROOT.synth.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode SYNTH.g
     678end
     679
     680macro cksynth
     681  if ($0 != 2)
     682    echo "USAGE: cksynth (stage)"
     683    break
     684  end
     685
     686  if (not($DO_SYNTH)) return
     687
     688  if ("$1" == "addstar")
     689    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2}
     690    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 3001)
     691    tapOK {ra[] == r_synth[]} "read in as many synth stars as we generated"
     692   
     693    match2d -closest ra dec r_synth d_synth 0.001 -index1 index1 -index2 index2
     694    subset tmp = index1 if (index1 >= 0)
     695    tapOK {tmp[] == ra[]} "synth sources match those generated"
     696  end
     697
     698  if ("$1" == "relphot")
     699    subset rs = r_synth if (m_synth < 13.5)
     700    subset ds = d_synth if (m_synth < 13.5)
     701    avmatch rs ds 0.1 g:flags
     702    subset tmp = rs if (g:flags & 0x04); # synth used for photometry
     703    tapOK {tmp[] == rs[]} "synth objects used for photometry properly marked"
     704  end
     705end
     706
     707macro mktycho
     708  if ($0 != 3)
     709    echo "USAGE: mktycho (fileroot) (catdir)"
     710    break
     711  end
     712
     713  if (not($DO_TYCHO)) return
     714
     715  local ROOT CATDIR
     716
     717  $ROOT    = $1
     718  $CATDIR  = $2
     719
     720  subset r_tycho  = stars_ra    if (stars_mag < 12)
     721  subset d_tycho  = stars_dec   if (stars_mag < 12)
     722  subset m_tycho  = stars_mag   if (stars_mag < 12)
     723  subset dm_tycho = stars_dmag  if (stars_mag < 12)
     724  subset fl_tycho = stars_flags if (stars_mag < 12)
     725
     726  write $ROOT.tycho.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" r_tycho d_tycho m_tycho dm_tycho fl_tycho
     727
     728  tapEXEC addstar -ref $ROOT.tycho.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode TYCHO_B
     729
     730  $KAPA = kapa -noX
     731  dev -n tmpdev
     732  # resize 100 100
     733  resize 50 50
     734  region $RA_CENTER $DEC_CENTER 0.2
     735  section default -imtool none
     736  # causes trouble for this to come before region...
     737  create tmp 0 1
     738  cdensify tmpmap tmp tmp
     739  set map = zero(tmpmap) + 0.2
     740  close
     741  $KAPA = kapa
     742
     743  local filter 
     744  foreach filter g r i z y
     745    if ("$filter" == "g")
     746      wd map $ROOT.synthzpts.fits -bitpix -32 -bzero 0.0 -bscale 1.0
     747    end
     748    keyword map EXTNAME -w "map_$filter"
     749    wd map $ROOT.synthzpts.fits -bitpix -32 -bzero 0.0 -bscale 1.0 -extend
     750  end
     751end
     752
     753macro cktycho
     754  if ($0 != 2)
     755    echo "USAGE: cktycho (stage)"
     756    break
     757  end
     758
     759  if (not($DO_TYCHO)) return
     760
     761  if ("$1" == "addstar")
     762    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2}
     763    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 2020)
     764    tapOK {ra[] == r_tycho[]} "read in as many tycho stars as we generated"
     765   
     766    match2d -closest ra dec r_tycho d_tycho 0.001 -index1 index1 -index2 index2
     767    subset tmp = index1 if (index1 >= 0)
     768    tapOK {tmp[] == ra[]} "tycho sources match those generated"
     769  end
     770
     771  if ("$1" == "relphot")
     772    subset rst = r_tycho if (m_tycho < 12)
     773    subset dst = d_tycho if (m_tycho < 12)
     774    # XXX : should not restrict to 'g'
     775    avmatch rst dst 0.1 g:flags
     776    subset tmp = rst if (g:flags & 0x40); # tycho used for photometry
     777    tapOK {tmp[] == rst[]} "tycho objects used for photometry properly marked"
     778
     779    # non-tycho stars with synth phot should be repaired
     780    subset rsn = r_synth if (m_synth > 12) && (m_synth < 13.5)
     781    subset dsn = d_synth if (m_synth > 12) && (m_synth < 13.5)
     782    # XXX : should not restrict to 'g'
     783    avmatch rsn dsn 0.1 g:flags
     784    subset tmp = rsn if (g:flags & 0x80); # synth photometry repaired
     785    tapOK {tmp[] == rsn[]} "tycho objects used for photometry properly marked"
     786  end
    442787end
    443788
    … …  
    456801
    457802  # define the season boundaries
    458   keyword phu TS0_0000 -wf 55000.0
    459   keyword phu TS1_0000 -wf 55010.0
    460   keyword phu TS0_0001 -wf 55010.0
    461   keyword phu TS1_0001 -wf 55020.0
    462   keyword phu TS0_0002 -wf 55020.0
    463   keyword phu TS1_0002 -wf 55030.0
     803  keyword phu S0_MJD0 -wf 55000.0
     804  keyword phu S1_MJD0 -wf 55010.0
     805  keyword phu S0_MJD1 -wf 55010.0
     806  keyword phu S1_MJD1 -wf 55020.0
     807  keyword phu S0_MJD2 -wf 55020.0
     808  keyword phu S1_MJD2 -wf 55030.0
    464809
    465810  # create a single flat-field correction map for a single chip
    466811  mcreate cell_off 2 2
    467   cell_off[0][0] =  0.01
    468   cell_off[1][0] = -0.01
    469   cell_off[0][1] =  0.02
    470   cell_off[1][1] = -0.02
     812  if ($USE_CELL_OFFSETS)
     813    cell_off[0][0] =  0.05
     814    cell_off[1][0] = -0.03
     815    cell_off[0][1] =  0.08
     816    cell_off[1][1] = -0.10
     817  end
    471818
    472819  # in the correction table, we have an image of the full exposure
    … …  
    481828    for iyc 0 $NCELL_Y
    482829      for ix 0 $NCHIP_X
     830        set tcell_off = cell_off
     831        if ($ix < 4)
     832          rotate tcell_off 180
     833        end
    483834        for ixc 0 $NCELL_X
    484           concat cell_off[$ixc][$iyc] image_offset
     835          concat tcell_off[$ixc][$iyc] image_offset
    485836        end
    486837      end
    … …  
    498849  set mjd_obs = mjd_uc
    499850  set zp = zpt_uc
    500   set resid = 0.02*rnd(zpt_uc) - 0.01
     851
     852  if ($SET_UC_RESID)
     853    set resid = 0.02*rnd(zpt_uc) - 0.01
     854  else
     855    set resid = zero(zpt_uc)
     856  end
    501857
    502858  delete -q flatcorr

  • branches/eam_branches/ohana.20150429/src/addstar/test/simple.dvo

    r38347 r38348  
    5454
    5555  mkinput
    56   echo mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
    57   exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
    58   if ($TAP_VERBOSE)
    59     echo exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
    60     exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
    61   else
    62     echo addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
    63     exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass >& tmp.log
    64   end
    65 
    66   echo list testfields -copy testfields_$1
     56  tapEXEC mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
     57  tapEXEC addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
     58
    6759  list testfields -copy testfields_$1
    6860
    … …  
    9486  set d = $name:0 - $name:2
    9587  vstat -q d
    96   tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN)"
    97   tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
     88  tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN = $MEAN)"
     89  tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $SIGMA)"
    9890
    9991  # detection ID must be first for this to work:
    … …  
    144136    if ("$name:0" == "PSF_THETA")
    145137      # echo $MEAN
    146       tapOK {abs($MEAN)  < 0.006} "$name:0 vs $name:2 (MEAN)"
    147       tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
     138      tapOK {abs($MEAN)  < 0.006} "$name:0 vs $name:2 (MEAN = $MEAN)"
     139      tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $SIGMA)"
    148140      continue
    149141    end
    150142
    151     tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN)"
    152     tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
     143    tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN = $MEAN)"
     144    tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $MEAN)"
    153145  end   
    154146

  • branches/eam_branches/ohana.20150429/src/addstar/test/tap.dvo

    r34405 r38348  
    33if (not($?TAP_BREAK)) set TAP_BREAK = 0
    44if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 0
     5
     6macro tapEXEC
     7  if ($0 < 3)
     8    echo "USAGE: tapEXEC (command)"
     9    break
     10  end
     11
     12  local i cmd result
     13
     14  $cmd = ""
     15  for i 1 $0
     16    $cmd = $cmd $$i
     17  end
     18
     19  if ($TAP_VERBOSE >= 2) echo $cmd
     20
     21  break -auto off
     22  exec $cmd >& tap.log
     23  $result = $STATUS
     24  break -auto on
     25
     26  if (not($result))
     27    exec cat tap.log
     28    echo "not ok : $cmd"
     29    $TAP_NFAIL ++
     30    $TAP_LAST = 0
     31    if ($TAP_BREAK)
     32     break
     33    end
     34  else
     35    if ($TAP_VERBOSE >= 1) echo "ok : $cmd"
     36    $TAP_LAST = 1
     37  end
     38  $TAP_NDONE ++
     39end
    540
    641macro tapOK
    … …  
    1146
    1247  if ($1)
    13     if ($TAP_VERBOSE)
     48    if ($TAP_VERBOSE >= 1)
    1449      echo "ok : $2"
    1550    end

  • branches/eam_branches/ohana.20150429/src/dvomerge/src/dvorepairDeleteImageList.c

    r38345 r38348  
    4646  FILE *cpmFile = NULL;
    4747
    48   char catformat;
     48  DVOCatFormat catformat;
    4949
    5050  N = get_argument (argc, argv, "-delete-image-list");

  • branches/eam_branches/ohana.20150429/src/dvomerge/src/dvorepairDeleteImagesByExternID.c

    r38346 r38348  
    3838  char *imageFilename = NULL;
    3939
    40   char catformat;
     40  DVOCatFormat catformat;
    4141
    4242  N = get_argument (argc, argv, "-delete-images-by-extern-id");
    … …  
    279279        gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    280280
     281        myAbort ("fix compression");
     282
    281283        // this function can only handle PS1_V5 and later formats
    282284        // NOTE: FtableToMeasure (and equivalent) free the data associated with cpmFtable.

  • branches/eam_branches/ohana.20150429/src/dvomerge/src/dvorepairFixCPT.c

    r37046 r38348  
    8686  Matrix matrix;
    8787
    88   char catformat;
     88  DVOCatFormat catformat;
    8989
    9090  cpmFtable.header = &cpmHeaderTBL;
    … …  
    120120  }
    121121
     122  myAbort ("fix cpts");
    122123  measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
    123124  myAssert(measure, "failed to convert ftable to measure data");

  • branches/eam_branches/ohana.20150429/src/dvomerge/src/dvorepairImagesVsMeasures.c

    r37046 r38348  
    3232  FILE *cpmFile = NULL;
    3333
    34   char catformat;
     34  DVOCatFormat catformat;
    3535
    3636  N = get_argument (argc, argv, "-images-vs-measures");

  • branches/eam_branches/ohana.20150429/src/libdvo/include/dvo.h

    r38346 r38348  
    974974DVOCatFormat FtableGetFormat (FTable *ftable);
    975975
    976 Measure *FtableToMeasure   (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format);
    977 Average *FtableToAverage   (FTable *ftable, off_t *Naverage,  DVOCatFormat *format, SecFilt **primary, char nativeBytes);
    978 SecFilt *FtableToSecFilt   (FTable *ftable, off_t *Nsecfilt,  DVOCatFormat *format);
    979 Lensing *FtableToLensing   (FTable *ftable, off_t *Nlensing,  DVOCatFormat *format);
    980 Lensobj *FtableToLensobj   (FTable *ftable, off_t *Nlensobj,  DVOCatFormat *format);
    981 StarPar *FtableToStarPar   (FTable *ftable, off_t *Nstarpar,  DVOCatFormat *format);
    982 GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format);
     976Measure *FtableToMeasure   (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeOrder);
     977Average *FtableToAverage   (FTable *ftable, off_t *Naverage,  DVOCatFormat *format, SecFilt **primary, char nativeOrder);
     978SecFilt *FtableToSecFilt   (FTable *ftable, off_t *Nsecfilt,  DVOCatFormat *format, char nativeOrder);
     979Lensing *FtableToLensing   (FTable *ftable, off_t *Nlensing,  DVOCatFormat *format, char nativeOrder);
     980Lensobj *FtableToLensobj   (FTable *ftable, off_t *Nlensobj,  DVOCatFormat *format, char nativeOrder);
     981StarPar *FtableToStarPar   (FTable *ftable, off_t *Nstarpar,  DVOCatFormat *format, char nativeOrder);
     982GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format, char nativeOrder);
    983983
    984984int      FtableToImage   (FTable *ftable, Header *theader, DVOCatFormat *format);
    985985
    986 int MeasureToFtable  (FTable *ftable, Average  *average,  Measure *measure, off_t Nmeasure, DVOCatFormat format);
     986int MeasureToFtable  (FTable *ftable, Average  *average,  Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative);
    987987int AverageToFtable  (FTable *ftable, Average  *average,  off_t Naverage,  DVOCatFormat format, SecFilt *primary, int swapFromNative);
    988 int SecFiltToFtable  (FTable *ftable, SecFilt  *secfilt,  off_t Nsecfilt,  DVOCatFormat format);
    989 int LensingToFtable  (FTable *ftable, Lensing  *lensing,  off_t Nlensing,  DVOCatFormat format);
    990 int LensobjToFtable  (FTable *ftable, Lensobj  *lensobj,  off_t Nlensobj,  DVOCatFormat format);
    991 int StarParToFtable  (FTable *ftable, StarPar  *starpar,  off_t Nstarpar,  DVOCatFormat format);
    992 int GalShapeToFtable (FTable *ftable, GalShape *galshape, off_t Ngalshape, DVOCatFormat format);
     988int SecFiltToFtable  (FTable *ftable, SecFilt  *secfilt,  off_t Nsecfilt,  DVOCatFormat format, int swapFromNative);
     989int LensingToFtable  (FTable *ftable, Lensing  *lensing,  off_t Nlensing,  DVOCatFormat format, int swapFromNative);
     990int LensobjToFtable  (FTable *ftable, Lensobj  *lensobj,  off_t Nlensobj,  DVOCatFormat format, int swapFromNative);
     991int StarParToFtable  (FTable *ftable, StarPar  *starpar,  off_t Nstarpar,  DVOCatFormat format, int swapFromNative);
     992int GalShapeToFtable (FTable *ftable, GalShape *galshape, off_t Ngalshape, DVOCatFormat format, int swapFromNative);
    993993
    994994int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format);

  • branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_create.c

    r38331 r38348  
    11# include <dvo.h>
    22# define DEBUG 0
     3
     4Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename);
    35
    46// create a new dvo catalog file (if split, lock extra files as well?)
    … …  
    911// catalog[0].lockmode
    1012void dvo_catalog_create (SkyRegion *region, Catalog *catalog) {
    11 
    12   int length;
    13   char *path, *root, *file, *line;
    14   time_t now;
    1513
    1614  if (DEBUG) fprintf (stderr, "new catalog file: %s\n", catalog[0].filename);
    … …  
    2927
    3028  if (catalog[0].catmode == DVO_MODE_SPLIT) {
    31     path = pathname (catalog[0].filename);
    32     root = filerootname (catalog[0].filename);
    33     length = strlen(path) + strlen(root) + 6;
    34 
    35     /* define measure catalog file */
    36     ALLOCATE (catalog[0].measure_catalog, Catalog, 1);
    37     dvo_catalog_init (catalog[0].measure_catalog, TRUE);
    38 
    39     /* create basic data for measure catalog file */
    40     gfits_create_header (&catalog[0].measure_catalog[0].header);
    41 
    42     ALLOCATE (catalog[0].measure_catalog[0].filename, char, length);
    43 
    44     sprintf (catalog[0].measure_catalog[0].filename, "%s/%s.cpm", path, root);
    45     file = filebasename (catalog[0].measure_catalog[0].filename);
    46     gfits_modify (&catalog[0].header, "MEASURE", "%s", 1, file);
    47     free (file);
    48 
    49     /* define missing catalog file */
    50     ALLOCATE (catalog[0].missing_catalog, Catalog, 1);
    51     dvo_catalog_init (catalog[0].missing_catalog, TRUE);
    52 
    53     /* create basic data for missing catalog file */
    54     gfits_create_header (&catalog[0].missing_catalog[0].header);
    55     ALLOCATE (catalog[0].missing_catalog[0].filename, char, length);
    56     sprintf (catalog[0].missing_catalog[0].filename, "%s/%s.cpn", path, root);
    57     file = filebasename (catalog[0].missing_catalog[0].filename);
    58     gfits_modify (&catalog[0].header, "MISSING", "%s", 1, file);
    59     free (file);
    60 
    61     /* define secfilt catalog file */
    62     ALLOCATE (catalog[0].secfilt_catalog, Catalog, 1);
    63     dvo_catalog_init (catalog[0].secfilt_catalog, TRUE);
    64 
    65     /* create basic data for secfilt catalog file */
    66     gfits_create_header (&catalog[0].secfilt_catalog[0].header);
    67     ALLOCATE (catalog[0].secfilt_catalog[0].filename, char, length);
    68     sprintf (catalog[0].secfilt_catalog[0].filename, "%s/%s.cps", path, root);
    69     file = filebasename (catalog[0].secfilt_catalog[0].filename);
    70     gfits_modify (&catalog[0].header, "SECFILT", "%s", 1, file);
    71     free (file);
    72 
    73     /* define lensing catalog file */
    74     ALLOCATE (catalog[0].lensing_catalog, Catalog, 1);
    75     dvo_catalog_init (catalog[0].lensing_catalog, TRUE);
    76 
    77     /* create basic data for lensing catalog file */
    78     gfits_create_header (&catalog[0].lensing_catalog[0].header);
    79     ALLOCATE (catalog[0].lensing_catalog[0].filename, char, length);
    80     sprintf (catalog[0].lensing_catalog[0].filename, "%s/%s.cpx", path, root);
    81     file = filebasename (catalog[0].lensing_catalog[0].filename);
    82     gfits_modify (&catalog[0].header, "LENSING", "%s", 1, file);
    83     free (file);
    84 
    85     /* define lensobj catalog file */
    86     ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1);
    87     dvo_catalog_init (catalog[0].lensobj_catalog, TRUE);
    88 
    89     /* create basic data for lensobj catalog file */
    90     gfits_create_header (&catalog[0].lensobj_catalog[0].header);
    91     ALLOCATE (catalog[0].lensobj_catalog[0].filename, char, length);
    92     sprintf (catalog[0].lensobj_catalog[0].filename, "%s/%s.cpy", path, root);
    93     file = filebasename (catalog[0].lensobj_catalog[0].filename);
    94     gfits_modify (&catalog[0].header, "LENSOBJ", "%s", 1, file);
    95     free (file);
    96 
    97     /* define starpar catalog file */
    98     ALLOCATE (catalog[0].starpar_catalog, Catalog, 1);
    99     dvo_catalog_init (catalog[0].starpar_catalog, TRUE);
    100 
    101     /* create basic data for starpar catalog file */
    102     gfits_create_header (&catalog[0].starpar_catalog[0].header);
    103     ALLOCATE (catalog[0].starpar_catalog[0].filename, char, length);
    104     sprintf (catalog[0].starpar_catalog[0].filename, "%s/%s.cpz", path, root);
    105     file = filebasename (catalog[0].starpar_catalog[0].filename);
    106     gfits_modify (&catalog[0].header, "STARPAR", "%s", 1, file);
    107     free (file);
    108 
    109     /* define galshape catalog file */
    110     ALLOCATE (catalog[0].galshape_catalog, Catalog, 1);
    111     dvo_catalog_init (catalog[0].galshape_catalog, TRUE);
    112 
    113     /* create basic data for galshape catalog file */
    114     gfits_create_header (&catalog[0].galshape_catalog[0].header);
    115     ALLOCATE (catalog[0].galshape_catalog[0].filename, char, length);
    116     sprintf (catalog[0].galshape_catalog[0].filename, "%s/%s.cpx", path, root);
    117     file = filebasename (catalog[0].galshape_catalog[0].filename);
    118     gfits_modify (&catalog[0].header, "GALSHAPE", "%s", 1, file);
    119     free (file);
    120 
    121     free (path);
    122     free (root);
     29    catalog[0].measure_catalog  = dvo_catalog_create_subcat (catalog, "cpm", "MEASURE");
     30    catalog[0].missing_catalog  = dvo_catalog_create_subcat (catalog, "cpn", "MISSING");
     31    catalog[0].secfilt_catalog  = dvo_catalog_create_subcat (catalog, "cps", "SECFILT");
     32    catalog[0].lensing_catalog  = dvo_catalog_create_subcat (catalog, "cpx", "LENSING");
     33    catalog[0].lensobj_catalog  = dvo_catalog_create_subcat (catalog, "cpy", "LENSOBJ");
     34    catalog[0].starpar_catalog  = dvo_catalog_create_subcat (catalog, "cpz", "STARPAR");
     35    catalog[0].galshape_catalog = dvo_catalog_create_subcat (catalog, "cpq", "GALSHAPE");
    12336
    12437    // lock the additional split files
    … …  
    16679
    16780  /* write creation date in header */
     81  time_t now;
    16882  ohana_str_to_time ("now", &now);
    169   line = ohana_sec_to_date (now);
     83  char *line = ohana_sec_to_date (now);
    17084  gfits_modify (&catalog[0].header, "DATE", "%s", 1, line);
    17185  free (line);
    … …  
    187101}
    188102 
     103Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename) {
     104 
     105  char *path = pathname (catalog[0].filename);
     106  char *root = filerootname (catalog[0].filename);
     107  int length = strlen(path) + strlen(root) + 6;
     108   
     109  Catalog *subcat;
     110 
     111  /* define subcat catalog file */
     112  ALLOCATE (subcat, Catalog, 1);
     113  dvo_catalog_init (subcat, TRUE);
     114 
     115  /* create basic data for measure catalog file */
     116  gfits_create_header (&subcat->header);
     117
     118  subcat->catcompress = catalog->catcompress;
     119
     120  ALLOCATE (subcat->filename, char, length);
     121
     122  sprintf (subcat->filename, "%s/%s.%s", path, root, ext);
     123
     124  char *file = filebasename (subcat->filename);
     125  gfits_modify (&catalog[0].header, tablename, "%s", 1, file);
     126
     127  free (file);
     128  free (path);
     129  free (root);
     130
     131  return subcat;
     132}
     133
    189134int dvo_catalog_set_range (Catalog *catalog) {
    190135

  • branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_mef.c

    r38345 r38348  
    105105      return (FALSE);
    106106    }
    107     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat);
     107    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat, FALSE);
    108108    if (Nmeasure != catalog[0].Nmeas_disk) {
    109109      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
    … …  
    163163    /* how many entries do we expect from the secfilt table? */
    164164    Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage;
    165     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
     165    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, FALSE);
    166166    if (Nexpect != Nitems) {
    167167      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nexpect,  Nitems);
    … …  
    216216      return (FALSE);
    217217    }
    218     catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat);
     218    catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat, FALSE);
    219219    if (Nlensing != catalog[0].Nlensing_disk) {
    220220      fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
    … …  
    241241      return (FALSE);
    242242    }
    243     catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat);
     243    catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE);
    244244    if (Nlensobj != catalog[0].Nlensobj_disk) {
    245245      fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
    … …  
    266266      return (FALSE);
    267267    }
    268     catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat);
     268    catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE);
    269269    if (Nstarpar != catalog[0].Nstarpar_disk) {
    270270      fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
    … …  
    291291      return (FALSE);
    292292    }
    293     catalog[0].galshape = FtableToGalShape (&ftable, &catalog[0].Ngalshape, &catalog[0].catformat);
     293    catalog[0].galshape = FtableToGalShape (&ftable, &catalog[0].Ngalshape, &catalog[0].catformat, FALSE);
    294294    if (Ngalshape != catalog[0].Ngalshape_disk) {
    295295      fprintf (stderr, "Warning: mismatch between Ngalshape in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalshape,  catalog[0].Ngalshape_disk);
    … …  
    398398
    399399  /* write out Measure table (convert to FITS table format) */
    400   MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat);
     400  MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat, TRUE);
    401401  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    402402    fprintf (stderr, "can't write table header");
    … …  
    425425  /* write out SecFilt table (convert to FITS table format) */
    426426  Nitems = catalog[0].Naverage * Nsecfilt;
    427   SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat);
     427  SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat, TRUE);
    428428  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    429429    fprintf (stderr, "can't write table header");
    … …  
    438438
    439439  /* write out Lensing table (convert to FITS table format) */
    440   LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat);
     440  LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat, TRUE);
    441441  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    442442    fprintf (stderr, "can't write table header");
    … …  
    451451
    452452  /* write out Lensobj table (convert to FITS table format) */
    453   LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat);
     453  LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat, TRUE);
    454454  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    455455    fprintf (stderr, "can't write table header");
    … …  
    464464
    465465  /* write out StarPar table (convert to FITS table format) */
    466   StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat);
     466  StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat, TRUE);
    467467  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    468468    fprintf (stderr, "can't write table header");
    … …  
    477477
    478478  /* write out GalShape table (convert to FITS table format) */
    479   GalShapeToFtable (&ftable, catalog[0].galshape, catalog[0].Ngalshape, catalog[0].catformat);
     479  GalShapeToFtable (&ftable, catalog[0].galshape, catalog[0].Ngalshape, catalog[0].catformat, TRUE);
    480480  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    481481    fprintf (stderr, "can't write table header");

  • branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c

    r38346 r38348  
    55// * empty (file is not found)
    66// * ok
     7
     8// utility function to see if we should compress this table)
     9int output_is_compressed (int start, int Nrows, int Ntotal, DVOCatCompress catcompress) {
     10  int fullWrite = (start == 0) && (Nrows == Ntotal);
     11  if (fullWrite && catcompress) return TRUE;  // we want to compress, so do not swap
     12  return FALSE;
     13}
    714
    815int dvo_catalog_secfilt_to_primary (Catalog *catalog, SecFilt **myPrimary, SecFilt **mySecfilt, int *myNsecfilt) {
    … …  
    159166 
    160167  int fullWrite = (start == 0) && (Nrows == Ntotal);
    161 
    162   if (fullWrite && catalog->catcompress) {
     168  int isCompressed = output_is_compressed (start, Nrows, Ntotal, catalog->catcompress);
     169
     170  if (isCompressed) {
     171    fprintf (stderr, "compress %s\n", catalog->filename);
    163172    char *compressMode = dvo_catalog_compress_string (catalog->catcompress);
    164173    if (!gfits_compress_table (ftable, &cmptable, 10, compressMode)) {
    … …  
    170179
    171180  // write the table data
    172   if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) {
    173     fprintf (stderr, "can't write table data\n");
    174     return (FALSE);
     181  if (fullWrite) {
     182    if (!gfits_fwrite_Theader (catalog->f, outtable->header)) {
     183      fprintf (stderr, "can't write table data\n");
     184      return (FALSE);
     185    }
     186    if (!gfits_fwrite_table (catalog->f, outtable)) {
     187      fprintf (stderr, "can't write table data\n");
     188      return (FALSE);
     189    }
     190  } else {
     191    if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) {
     192      fprintf (stderr, "can't write table data\n");
     193      return (FALSE);
     194    }
    175195  }
    176196  fullsize += outtable->datasize + outtable->header->datasize;
    … …  
    224244  free (path);
    225245
     246  // inherit compression mode for this subcat;
     247  subcat->catcompress = catalog->catcompress;
     248
    226249  /* lock & open catalog file */
    227250  status = dvo_catalog_lock (subcat, catalog[0].lockmode);
    … …  
    252275  }
    253276  return (DVO_CAT_OPEN_OK);
     277}
     278
     279// ftable must already exist and have a valid, loaded header
     280int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder) {
     281
     282  // fread_ftable_data requires the header
     283
     284  *nativeOrder = FALSE;
     285  if (!gfits_fread_ftable_data (catalog->f, ftable, FALSE)) {
     286    if (VERBOSE) fprintf (stderr, "can't read table data\n");
     287    return FALSE;
     288  }
     289  // NOTE: fread_ftable returns an unswapped table; uncompress swaps the bytes into native order
     290   
     291  if (gfits_extension_is_compressed_table (ftable->header)) {
     292    FTable rawtable;
     293    Header rawheader;
     294    gfits_init_table (&rawtable);
     295    gfits_init_header (&rawheader);
     296    rawtable.header = &rawheader;
     297
     298    // NOTE: uncompress swaps the bytes into native order
     299    if (!gfits_uncompress_table (ftable, &rawtable)) {
     300      if (VERBOSE) fprintf (stderr, "failed to uncompress table\n");
     301      gfits_free_table (ftable);
     302      return FALSE;
     303    }
     304    catalog->catcompress = DVO_COMPRESS_GZIP_1; // if any table is compressed, set all to a compress state?
     305
     306    // free the buffers
     307    gfits_free_header (ftable->header);
     308    gfits_free_table (ftable);
     309   
     310    // copies values but does not allocate new memory
     311    gfits_copy_header_ptr (&rawheader, ftable->header);
     312    gfits_copy_table_ptr (&rawtable, ftable);
     313    *nativeOrder = TRUE;
     314  }
     315  return TRUE;
    254316}
    255317
    … …  
    266328  off_t Nitems;
    267329  int status, Nsecfilt;
    268   Header header;
    269   FTable ftable;
    270330  SecFilt *primary;
    271331
    272   /* ftable header storage for below */
    273   ftable.header = &header;
    274   ftable.buffer = NULL;
    275   header.buffer = NULL;
    276332  primary = NULL;
    277333  status = FALSE;
    … …  
    331387  fseeko (catalog[0].f, Nbytes, SEEK_SET);
    332388
     389  /* ftable header storage for below */
     390  Header header;
     391  FTable ftable;
     392  gfits_init_table  (&ftable);
     393  gfits_init_header (&header);
     394  ftable.header = &header;
     395
    333396  /* read Average table header */
    334397  if (!gfits_fread_header (catalog[0].f, &header)) {
    … …  
    337400  }
    338401  if (catalog[0].catflags & LOAD_AVES) {
    339     Header rawheader;
    340     FTable rawtable;
    341     rawtable.header = &rawheader;
    342     FTable *outtable = &ftable;
    343402    char nativeOrder = FALSE;
    344 
    345     /* read Average table data : format is irrelevant here */
    346     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    347       if (VERBOSE) fprintf (stderr, "can't read table average data\n");
    348       return (FALSE);
    349     }
    350     if (gfits_extension_is_compressed_table (ftable.header)) {
    351       if (!gfits_uncompress_table (&ftable, &rawtable)) {
    352         if (VERBOSE) fprintf (stderr, "failed to uncompress table\n");
    353         return (FALSE);
    354       }
    355       gfits_free_table (&ftable);
    356       outtable = &rawtable;
    357       nativeOrder = TRUE;
    358       catalog->catcompress = DVO_COMPRESS_GZIP_1; // if any table is compressed, set all to a compress state?
    359     }
    360     // NOTE: fread_ftable returns an unswapped table; uncompress swaps the bytes into native order
     403    if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder)) {
     404      if (VERBOSE) fprintf (stderr, "can't read Average table\n");
     405      gfits_free_header (&header);
     406      return FALSE;
     407    }
    361408
    362409    /* convert the saved version of the table to the internal version.  Old versions of DVO stored
    363410     * one of the average magnitudes in Average.  We save this in case it is needed below.  NOTE:
    364411     * primary is only used if we read in the secfilt table, otherwise it should be freed */
    365     catalog[0].average = FtableToAverage (outtable, &Naverage, &catalog[0].catformat, &primary, nativeOrder);
     412    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, nativeOrder);
    366413    if (Naverage != catalog[0].Naves_disk) {
    367414      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naves_disk);
    368415    }
    369     gfits_free_header (&header);
    370416    catalog[0].Naverage = catalog[0].Naves_disk;
    371417    catalog[0].Naves_off = 0;
    … …  
    376422    catalog[0].Naves_off = catalog[0].Naves_disk;
    377423  }
     424  gfits_free_header (&header);
    378425
    379426  /*** Measure Table ***/
     427  gfits_init_table  (&ftable);
     428  gfits_init_header (&header);
     429  ftable.header = &header;
    380430  if (!(catalog[0].catflags & SKIP_MEAS)) {
    381431    // unless we specify 'skip', we still need to load the
    … …  
    389439  }
    390440  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) {
    391     // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK?
     441    char nativeOrder = FALSE;
    392442    /* read Measure table data */
    393     if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable, FALSE)) {
     443    if (!gfits_fread_uncompressed (catalog[0].measure_catalog, &ftable, &nativeOrder)) {
    394444      if (VERBOSE) fprintf (stderr, "can't read table measure data\n");
     445      gfits_free_header (&header);
    395446      return (FALSE);
    396447    }
    397448    /* convert data format to internal : returns number of row read in Nmeasure */
    398     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat);
     449    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, nativeOrder);
    399450    if (Nmeasure != catalog[0].Nmeas_disk) {
    400451      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
    … …  
    419470
    420471  /*** Missing Table ***/
     472  gfits_init_table  (&ftable);
     473  gfits_init_header (&header);
     474  ftable.header = &header;
    421475  status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE);
    422476  if (status == DVO_CAT_OPEN_FAIL) {
    … …  
    427481  }
    428482  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) {
     483    char nativeOrder = FALSE;
    429484    /* read Missing table data */
    430     if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable, FALSE)) {
     485    if (!gfits_fread_uncompressed (catalog[0].missing_catalog, &ftable, &nativeOrder)) {
    431486      if (VERBOSE) fprintf (stderr, "can't read table missing data\n");
    432487      return (FALSE);
    433488    }
    434489    /* no conversions currently defined : this just does the byte swap */
    435     catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, NULL);
     490    catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, &nativeOrder);
    436491    if (!catalog[0].missing) {
    437492      fprintf (stderr, "ERROR: failed to read missing\n");
    … …  
    453508
    454509  /*** Secfilt Table ***/
     510  gfits_init_table  (&ftable);
     511  gfits_init_header (&header);
     512  ftable.header = &header;
    455513  status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE);
    456514  // fprintf (stderr, "secfilt header %s : %zx\n", catalog[0].secfilt_catalog[0].filename, (size_t) catalog[0].secfilt_catalog[0].header.buffer);
    … …  
    462520  }
    463521  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) {
     522    char nativeOrder = FALSE;
    464523    /* read secfilt table data */
    465     if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable, FALSE)) {
     524    if (!gfits_fread_uncompressed (catalog[0].secfilt_catalog, &ftable, &nativeOrder)) {
    466525      if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n");
    467526      return (FALSE);
    468527    }
    469     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
     528    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, nativeOrder);
    470529    if (Nitems != catalog[0].Nsecf_disk) {
    471530      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  catalog[0].Nsecf_disk);
    … …  
    493552
    494553  /*** Lensing Table ***/
     554  gfits_init_table  (&ftable);
     555  gfits_init_header (&header);
     556  ftable.header = &header;
    495557  if (!(catalog[0].catflags & SKIP_LENSING)) {
    496558    // unless we specify 'skip', we still need to load the Lensing table
    … …  
    505567  }
    506568  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSING)) {
     569    char nativeOrder = FALSE;
    507570    // only read the Lensing table if the file actually exists
    508571    /* read Lensing table data */
    509     if (!gfits_fread_ftable_data (catalog[0].lensing_catalog[0].f, &ftable, FALSE)) {
     572    if (!gfits_fread_uncompressed (catalog[0].lensing_catalog, &ftable, &nativeOrder)) {
    510573      if (VERBOSE) fprintf (stderr, "can't read table lensing data\n");
    511574      return (FALSE);
    512575    }
    513576    /* convert data format to internal : returns number of row read in Nlensing */
    514     catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat);
     577    catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat, nativeOrder);
    515578    if (Nlensing != catalog[0].Nlensing_disk) {
    516579      fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
    … …  
    533596
    534597  /*** Lensobj Table ***/
     598  gfits_init_table  (&ftable);
     599  gfits_init_header (&header);
     600  ftable.header = &header;
    535601  if (!(catalog[0].catflags & SKIP_LENSOBJ)) {
    536602    // unless we specify 'skip', we still need to load the
    … …  
    545611  }
    546612  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSOBJ)) {
     613    char nativeOrder = FALSE;
    547614    // only read the Lensobj table if the file actually exists
    548615    /* read Lensobj table data */
    549     if (!gfits_fread_ftable_data (catalog[0].lensobj_catalog[0].f, &ftable, FALSE)) {
     616    if (!gfits_fread_uncompressed (catalog[0].lensobj_catalog, &ftable, &nativeOrder)) {
    550617      if (VERBOSE) fprintf (stderr, "can't read table lensobj data\n");
    551618      return (FALSE);
    552619    }
    553620    /* convert data format to internal : returns number of row read in Nlensobj */
    554     catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat);
     621    catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat, FALSE);
    555622    if (Nlensobj != catalog[0].Nlensobj_disk) {
    556623      fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
    … …  
    573640
    574641  /*** StarPar Table ***/
     642  gfits_init_table  (&ftable);
     643  gfits_init_header (&header);
     644  ftable.header = &header;
    575645  if (!(catalog[0].catflags & SKIP_STARPAR)) {
    576646    // unless we specify 'skip', we still need to load the
    … …  
    585655  }
    586656  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_STARPAR)) {
     657    char nativeOrder = FALSE;
    587658    // only read the StarPar table if the file actually exists
    588659    /* read StarPar table data */
    589     if (!gfits_fread_ftable_data (catalog[0].starpar_catalog[0].f, &ftable, FALSE)) {
     660    if (!gfits_fread_uncompressed (catalog[0].starpar_catalog, &ftable, &nativeOrder)) {
    590661      if (VERBOSE) fprintf (stderr, "can't read table starpar data\n");
    591662      return (FALSE);
    592663    }
    593664    /* convert data format to internal : returns number of row read in Nstarpar */
    594     catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat);
     665    catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat, FALSE);
    595666    if (Nstarpar != catalog[0].Nstarpar_disk) {
    596667      fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
    … …  
    613684
    614685  /*** GalShape Table ***/
     686  gfits_init_table  (&ftable);
     687  gfits_init_header (&header);
     688  ftable.header = &header;
    615689  if (!(catalog[0].catflags & SKIP_GALSHAPE)) {
    616690    // unless we specify 'skip', we still need to load the GalShape table
    … …  
    625699  }
    626700  if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_GALSHAPE)) {
     701    char nativeOrder = FALSE;
    627702    // only read the GalShape table if the file actually exists
    628703    /* read GalShape table data */
    629     if (!gfits_fread_ftable_data (catalog[0].galshape_catalog[0].f, &ftable, FALSE)) {
     704    if (!gfits_fread_uncompressed (catalog[0].galshape_catalog, &ftable, &nativeOrder)) {
    630705      if (VERBOSE) fprintf (stderr, "can't read table galshape data\n");
    631706      return (FALSE);
    632707    }
    633708    /* convert data format to internal : returns number of row read in Ngalshape */
    634     catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat);
     709    catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat, FALSE);
    635710    if (Ngalshape != catalog[0].Ngalshape_disk) {
    636711      fprintf (stderr, "Warning: mismatch between Ngalshape in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalshape,  catalog[0].Ngalshape_disk);
    … …  
    720795
    721796    Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt;
    722     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
     797    catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat, FALSE);
    723798    if (Nitems != Nexpect) {
    724799      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  Nexpect);
    … …  
    756831
    757832    /* convert data format to internal : returns number of row read in Nmeasure */
    758     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat);
     833    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, FALSE);
    759834    if (Nmeasure != Nrows) {
    760835      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    821896
    822897    /* convert data format to internal : returns number of row read in Nlensing */
    823     catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat);
     898    catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat, FALSE);
    824899    if (Nlensing != Nrows) {
    825900      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    852927
    853928    /* convert data format to internal : returns number of row read in Nlensobj */
    854     catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat);
     929    catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat, FALSE);
    855930    if (Nlensobj != Nrows) {
    856931      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    883958
    884959    /* convert data format to internal : returns number of row read in Nstarpar */
    885     catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat);
     960    catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat, FALSE);
    886961    if (Nstarpar != Nrows) {
    887962      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    914989
    915990    /* convert data format to internal : returns number of row read in Ngalshape */
    916     catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat);
     991    catalog[0].galshape = FtableToGalShape (&ftable, &Ngalshape, &catalog[0].catformat, FALSE);
    917992    if (Ngalshape != Nrows) {
    918993      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    10031078   
    10041079    // if we are going to compress, we need to receive unswapped data --
    1005     // gfits_compress_table expectes a table with unswapped data values
    1006     int swapFromNative = TRUE;
    1007     int fullWrite = (start == 0) && (Nrows == Naves_disk_new);
    1008     if (fullWrite && catalog->catcompress) {
    1009       swapFromNative = FALSE;
    1010     }
     1080    int swapFromNative = !output_is_compressed (start, Nrows, Naves_disk_new, catalog->catcompress);
    10111081
    10121082    if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) {
    … …  
    10461116    assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
    10471117
     1118    // if we are going to compress, we need to receive unswapped data --
     1119    int swapFromNative = !output_is_compressed (start, Nrows, Nmeas_disk_new, catalog->catcompress);
     1120
    10481121    // convert to external table format (note that the block above does not damage or free catalog.average)
    1049     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
     1122    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
    10501123      fprintf (stderr, "trouble converting format\n");
    10511124      goto failure;
    … …  
    10691142    }
    10701143
     1144    // if we are going to compress, we need to receive unswapped data --
     1145    int swapFromNative = !output_is_compressed (0, catalog->Nmissing, catalog->Nmissing, catalog->catcompress);
     1146
    10711147    // convert to external table format
    1072     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
     1148    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
    10731149      fprintf (stderr, "trouble converting format\n");
    10741150      goto failure;
    … …  
    10981174    // XXX check these for consistency...
    10991175
     1176    // if we are going to compress, we need to receive unswapped data --
     1177    int swapFromNative = !output_is_compressed (start, Nrows, Nsecf_disk_new, catalog->catcompress);
     1178
    11001179    // convert to external table format
    1101     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
     1180    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative);
    11021181
    11031182    // write out SecFilt table
    … …  
    11231202
    11241203    if (catalog[0].Nlensing) {
     1204      // if we are going to compress, we need to receive unswapped data --
     1205      int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
     1206
    11251207      // convert to external table format (note that the block above does not damage or free catalog.average)
    1126       if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
     1208      if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
    11271209        fprintf (stderr, "trouble converting format\n");
    11281210        goto failure;
    … …  
    11521234
    11531235    if (catalog[0].Nlensobj) {
     1236      // if we are going to compress, we need to receive unswapped data --
     1237      int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
     1238
    11541239      // convert to external table format (note that the block above does not damage or free catalog.average)
    1155       if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
     1240      if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
    11561241        fprintf (stderr, "trouble converting format\n");
    11571242        goto failure;
    … …  
    11811266
    11821267    if (catalog[0].Nstarpar) {
     1268      // if we are going to compress, we need to receive unswapped data --
     1269      int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
     1270
    11831271      // convert to external table format (note that the block above does not damage or free catalog.average)
    1184       if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
     1272      if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
    11851273        fprintf (stderr, "trouble converting format\n");
    11861274        goto failure;
    … …  
    12101298
    12111299    if (catalog[0].Ngalshape) {
     1300      // if we are going to compress, we need to receive unswapped data --
     1301      int swapFromNative = !output_is_compressed (start, Nrows, Ngalshape_disk_new, catalog->catcompress);
     1302
    12121303      // convert to external table format (note that the block above does not damage or free catalog.average)
    1213       if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat)) {
     1304      if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat, swapFromNative)) {
    12141305        fprintf (stderr, "trouble converting format\n");
    12151306        goto failure;
    … …  
    13861477
    13871478    // convert to external table format
    1388     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
     1479    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
    13891480      fprintf (stderr, "trouble converting format\n");
    13901481      goto failure;
    … …  
    14381529
    14391530    // convert to external table format
    1440     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
     1531    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
    14411532
    14421533    // write out SecFilt table
    … …  
    14621553
    14631554    // convert to external table format
    1464     if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
     1555    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
    14651556      fprintf (stderr, "trouble converting format\n");
    14661557      goto failure;
    … …  
    14891580
    14901581    // convert to external table format
    1491     if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
     1582    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
    14921583      fprintf (stderr, "trouble converting format\n");
    14931584      goto failure;
    … …  
    15161607
    15171608    // convert to external table format
    1518     if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
     1609    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
    15191610      fprintf (stderr, "trouble converting format\n");
    15201611      goto failure;
    … …  
    15431634
    15441635    // convert to external table format
    1545     if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat)) {
     1636    if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat, TRUE)) {
    15461637      fprintf (stderr, "trouble converting format\n");
    15471638      goto failure;
    … …  
    16831774    // convert to external table format
    16841775    // XXX does catalog.measure have averef correctly set up?
    1685     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
     1776    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
    16861777      fprintf (stderr, "trouble converting format\n");
    16871778      goto failure;
    … …  
    17361827
    17371828    // convert to external table format
    1738     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
     1829    SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
    17391830
    17401831    // write out SecFilt table
    … …  
    17611852    // convert to external table format
    17621853    // XXX does catalog.lensing have averef correctly set up?
    1763     if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
     1854    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
    17641855      fprintf (stderr, "trouble converting format\n");
    17651856      goto failure;
    … …  
    17891880    // convert to external table format
    17901881    // XXX does catalog.lensobj have averef correctly set up?
    1791     if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
     1882    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
    17921883      fprintf (stderr, "trouble converting format\n");
    17931884      goto failure;
    … …  
    18171908    // convert to external table format
    18181909    // XXX does catalog.starpar have averef correctly set up?
    1819     if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
     1910    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
    18201911      fprintf (stderr, "trouble converting format\n");
    18211912      goto failure;
    … …  
    18451936    // convert to external table format
    18461937    // XXX does catalog.galshape have averef correctly set up?
    1847     if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat)) {
     1938    if (!GalShapeToFtable (&ftable, &catalog[0].galshape[first], Nrows, catalog[0].catformat, TRUE)) {
    18481939      fprintf (stderr, "trouble converting format\n");
    18491940      goto failure;

  • branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_convert.c

    r38346 r38348  
    152152  switch (format) {
    153153    case DVO_FORMAT_INTERNAL: {
    154       gfits_table_set_Average (ftable, average, Naverage, TRUE);
     154      gfits_table_set_Average (ftable, average, Naverage, swapFromNative);
    155155      break; }
    156156
    … …  
    180180
    181181// FtableToMeasure needs the Average since old formats stored measure.dR,dD only
    182 Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format) {
     182Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeBytes) {
    183183
    184184  Measure *measure;
    … …  
    269269
    270270// MeasureToFtable needs the Average since old formats stored measure.dR,dD only
    271 int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format) {
     271int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative) {
    272272
    273273# define FORMAT_CASE(FORMAT, TYPE, ABS_COORDS)          \
    … …  
    276276      myAssert (ABS_COORDS || average, "conversion from internal needs average table"); \
    277277      tmpMeasure = MeasureInternalTo_##TYPE (average, measure, Nmeasure); \
    278         gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure, TRUE); \
     278        gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure, swapFromNative); \
    279279      free (tmpMeasure); \
    280280      break; }
    … …  
    283283  switch (format) {
    284284    case DVO_FORMAT_INTERNAL: {
    285       gfits_table_set_Measure (ftable, measure, Nmeasure, TRUE);
     285      gfits_table_set_Measure (ftable, measure, Nmeasure, swapFromNative);
    286286      break; }
    287287
    … …  
    310310/*** SecFilt / FTable conversion functions ***/
    311311
    312 SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, DVOCatFormat *format) {
     312SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) {
    313313
    314314  SecFilt *secfilt;
    … …  
    380380}
    381381
    382 int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format) {
     382int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative) {
    383383
    384384# define FORMAT_CASE(FORMAT, TYPE) \
    … …  
    386386      SecFilt_##TYPE *tmpSecFilt; \
    387387      tmpSecFilt = SecFiltInternalTo_##TYPE (secfilt, Nsecfilt); \
    388         gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt, TRUE); \
     388        gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt, swapFromNative); \
    389389      free (tmpSecFilt); \
    390390      break; }
    … …  
    393393  switch (format) {
    394394    case DVO_FORMAT_INTERNAL: {
    395       gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt, TRUE);
     395      gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt, swapFromNative);
    396396      break; }
    397397
    … …  
    420420/*** Lensing / FTable conversion functions ***/
    421421
    422 Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, DVOCatFormat *format) {
     422Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) {
    423423
    424424  Lensing *lensing;
    … …  
    520520
    521521// LensingToFtable needs the Average since old formats stored lensing.dR,dD only
    522 int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format) {
     522int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative) {
    523523
    524524# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    526526      Lensing_##TYPE *tmpLensing; \
    527527      tmpLensing = LensingInternalTo_##TYPE (lensing, Nlensing); \
    528         gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing, TRUE); \
     528        gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing, swapFromNative); \
    529529      free (tmpLensing); \
    530530      break; }
    … …  
    533533  switch (format) {
    534534    case DVO_FORMAT_INTERNAL: {
    535       gfits_table_set_Lensing (ftable, lensing, Nlensing, TRUE);
     535      gfits_table_set_Lensing (ftable, lensing, Nlensing, swapFromNative);
    536536      break; }
    537537
    … …  
    559559/*** Lensobj / FTable conversion functions ***/
    560560
    561 Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, DVOCatFormat *format) {
     561Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) {
    562562
    563563  Lensobj *lensobj;
    … …  
    635635
    636636// LensobjToFtable needs the Average since old formats stored lensobj.dR,dD only
    637 int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format) {
     637int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative) {
    638638
    639639# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    641641      Lensobj_##TYPE *tmpLensobj; \
    642642      tmpLensobj = LensobjInternalTo_##TYPE (lensobj, Nlensobj); \
    643         gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj, TRUE); \
     643        gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj, swapFromNative); \
    644644      free (tmpLensobj); \
    645645      break; }
    … …  
    648648  switch (format) {
    649649    case DVO_FORMAT_INTERNAL: {
    650       gfits_table_set_Lensobj (ftable, lensobj, Nlensobj, TRUE);
     650      gfits_table_set_Lensobj (ftable, lensobj, Nlensobj, swapFromNative);
    651651      break; }
    652652
    … …  
    674674/*** StarPar / FTable conversion functions ***/
    675675
    676 StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, DVOCatFormat *format) {
     676StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) {
    677677
    678678  StarPar *starpar;
    … …  
    738738
    739739// StarParToFtable needs the Average since old formats stored starpar.dR,dD only
    740 int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format) {
     740int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative) {
    741741
    742742# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    744744      StarPar_##TYPE *tmpStarPar; \
    745745      tmpStarPar = StarParInternalTo_##TYPE (starpar, Nstarpar); \
    746         gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar, TRUE); \
     746        gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar, swapFromNative); \
    747747      free (tmpStarPar); \
    748748      break; }
    … …  
    751751  switch (format) {
    752752    case DVO_FORMAT_INTERNAL: {
    753       gfits_table_set_StarPar (ftable, starpar, Nstarpar, TRUE);
     753      gfits_table_set_StarPar (ftable, starpar, Nstarpar, swapFromNative);
    754754      break; }
    755755
    … …  
    778778/*** GalShape / FTable conversion functions ***/
    779779
    780 GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format) {
     780GalShape *FtableToGalShape (FTable *ftable, off_t *Ngalshape, DVOCatFormat *format, char nativeBytes) {
    781781
    782782  GalShape *galshape;
    … …  
    842842
    843843// GalShapeToFtable needs the Average since old formats stored galshape.dR,dD only
    844 int GalShapeToFtable (FTable *ftable, GalShape *galshape, off_t Ngalshape, DVOCatFormat format) {
     844int GalShapeToFtable (FTable *ftable, GalShape *galshape, off_t Ngalshape, DVOCatFormat format, int swapFromNative) {
    845845
    846846# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    848848      GalShape_##TYPE *tmpGalShape; \
    849849      tmpGalShape = GalShapeInternalTo_##TYPE (galshape, Ngalshape); \
    850         gfits_table_set_GalShape_##TYPE (ftable, tmpGalShape, Ngalshape, TRUE); \
     850        gfits_table_set_GalShape_##TYPE (ftable, tmpGalShape, Ngalshape, swapFromNative); \
    851851      free (tmpGalShape); \
    852852      break; }
    … …  
    855855  switch (format) {
    856856    case DVO_FORMAT_INTERNAL: {
    857       gfits_table_set_GalShape (ftable, galshape, Ngalshape, TRUE);
     857      gfits_table_set_GalShape (ftable, galshape, Ngalshape, swapFromNative);
    858858      break; }
    859859

  • branches/eam_branches/ohana.20150429/src/libfits/header/F_copy_H.c

    r27435 r38348  
    3030  return (TRUE);
    3131}       
     32
     33int gfits_copy_header_ptr (Header *in, Header *out) {
     34
     35  int i;
     36
     37  if (!in)  return FALSE;
     38  if (!out) return FALSE;
     39
     40  out[0].simple = in[0].simple;
     41  out[0].unsign = in[0].unsign;
     42  out[0].extend = in[0].extend;
     43  out[0].bitpix = in[0].bitpix;
     44
     45  out[0].Naxes  = in[0].Naxes;
     46  for (i = 0; i < FT_MAX_NAXES; i++)
     47    out[0].Naxis[i] = in[0].Naxis[i];
     48
     49  out[0].datasize = in[0].datasize;
     50
     51  out[0].pcount = in[0].pcount;
     52  out[0].gcount = in[0].gcount;
     53  out[0].bzero  = in[0].bzero;
     54  out[0].bscale = in[0].bscale;
     55
     56  out[0].buffer = in[0].buffer;
     57
     58  return (TRUE);
     59}       

  • branches/eam_branches/ohana.20150429/src/libfits/include/gfitsio.h

    r38340 r38348  
    271271int     gfits_uncompress_table         PROTO((FTable *srctable, FTable *tgttable));
    272272
     273// XXX EAM for testing
     274int gfits_dump_raw_table (FTable *table, char *message);
     275int gfits_dump_cmp_table (FTable *table, char *message);
     276
     277
    273278#endif /* FITSIO */

  • branches/eam_branches/ohana.20150429/src/libfits/table/F_compress_T.c

    r38343 r38348  
    137137  ALLOCATE (zdata, char, Nzdata_alloc);
    138138
     139  gfits_dump_raw_table (srctable, "cmp");
     140
    139141  // compress the data : copy into a tile, compress the tile, then add to the output table
    140142  // each tile -> 1 row of the output table
    … …  
    171173    if (!gfits_varlength_column_finish (tgttable, &fields[i].zdef)) ESCAPE(A);
    172174  }
     175
     176  gfits_dump_cmp_table (tgttable, "cmp");
    173177
    174178  free (raw);

  • branches/eam_branches/ohana.20150429/src/libfits/table/F_copy_T.c

    r38329 r38348  
    1212  ALLOCATE (out[0].buffer, char, out[0].datasize);
    1313  memcpy (out[0].buffer, in[0].buffer, out[0].datasize);
     14  return (TRUE);
     15}       
     16
     17/*********************** copy ftable pointers (does not copy memory) ***********************************/
     18int gfits_copy_ftable_ptr (FTable *in, FTable *out) {
     19
     20  if (!in)  return FALSE;
     21  if (!out) return FALSE;
     22
     23  /* find buffer size */
     24  out[0].validsize  = in[0].validsize;
     25  out[0].datasize   = in[0].datasize;
     26  out[0].heap_start = in[0].heap_start;
     27  out[0].header     = in[0].header;
     28  out[0].buffer     = in[0].buffer;
    1429  return (TRUE);
    1530}       

  • branches/eam_branches/ohana.20150429/src/libfits/table/F_uncompress_T.c

    r38343 r38348  
    66
    77int gfits_distribute_table_data (FTable *table, char *raw, int col, int row_start, int Nrows, int rowsize);
     8
     9# define VERBOSE_DUMP 0
     10
     11int gfits_dump_raw_table (FTable *table, char *message) {
     12# if (VERBOSE_DUMP)
     13  int i;
     14  fprintf (stderr, "%s data: ", message);
     15  // for (i = 0; i < table->header->Naxis[0]*table->header->Naxis[1]; i++) {
     16  for (i = 0; i < 16; i++) {
     17    fprintf (stderr, "0x%02hhx ", table->buffer[i]);
     18  }
     19  fprintf (stderr, "\n");
     20# endif
     21  return TRUE;
     22}
     23
     24int gfits_dump_cmp_table (FTable *table, char *message) {
     25# if (VERBOSE_DUMP)
     26  int i;
     27  fprintf (stderr, "%s pntr: ", message);
     28  for (i = 0; (i < 16) && (i < table->header->Naxis[0]*table->header->Naxis[1]); i++) {
     29    fprintf (stderr, "0x%02hhx ", table->buffer[i]);
     30  }
     31  fprintf (stderr, "\n");
     32
     33  fprintf (stderr, "%s data: ", message);
     34  for (i = 0; (i < 16) && (i < table->header->pcount); i++) {
     35    fprintf (stderr, "0x%02hhx ", table->buffer[table->heap_start + i]);
     36  }
     37  fprintf (stderr, "\n");
     38# endif
     39  return TRUE;
     40}
    841
    942int gfits_uncompress_table (FTable *srctable, FTable *tgttable) {
    … …  
    105138  // each tile -> 1 row of the output table
    106139 
     140  gfits_dump_cmp_table (srctable, "unc");
     141
    107142  off_t row;
    108143  for (row = 0; row < srcheader->Naxis[1]; row++) {
    … …  
    129164      }
    130165
     166      // int valid = (row == 0) && (i == 0);
     167
    131168      off_t row_start = row*ztilelen;
    132169      // ^- first row for this tile & field
    … …  
    140177    }
    141178  }
     179
     180  gfits_dump_raw_table (tgttable, "unc");
    142181
    143182  free (raw);
Note: See TracChangeset for help on using the changeset viewer.