Index: trunk/ippToPsps/test/mkgpc1data.dr2.dvo
===================================================================
--- trunk/ippToPsps/test/mkgpc1data.dr2.dvo	(revision 40250)
+++ trunk/ippToPsps/test/mkgpc1data.dr2.dvo	(revision 40250)
@@ -0,0 +1,785 @@
+#!/usr/bin/env dvo
+# -*-sh-*-
+input tap.dvo
+
+if (not($?VERBOSE)) set VERBOSE = 0
+if (not($?PARALLEL)) set PARALLEL = 0
+
+$SMALLTEST = 1
+
+## this script must be run in this directory, output data goes to 'testdata'
+$OUTDIR = testdata
+
+# note that ipptopsps and gpc1db need the path ex extension
+# I'm setting the base filename without the .smf and add it in mkcatdir.*
+
+# some global value for ease of access:
+$RA = 10.0
+$DEC = 20.0
+$SKYCELL = skycell.1133.081
+$TESS_ID = RINGS.V3
+$BAD_PSFQF_FRAC = 0.0
+$ADDNOISE = "-no-noise"
+# $ADDNOISE = ""
+
+$CATFORMAT = PS1_V6
+$CMFFORMAT = PS1_V5
+$DIFFORMAT = PS1_DV5
+
+
+# a note on double quotes and TESS_ID: 
+# since the variables TESS_ID and SKYCELL are used below to construct the list, their values should NOT be wrapped in double quotes
+# or those double quotes are embedded in the strings in the list
+# when TESS_ID and SKYCELL appear in mysql commands as values they need to be protected with double quotes so mysql does not 
+# interpret them as db table fields
+
+# imagedata describes the full fake exposure set.  the fields are
+list imagedata
+# exp_id stk_id filebase          M_off date       time      warpfile                       filter  difffile
+   1 	 1      $OUTDIR/test.cam  0.000 2010/01/01 01:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   2 	 1      $OUTDIR/test.cam -0.025 2010/01/01 02:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   3 	 1      $OUTDIR/test.cam  0.025 2010/01/01 03:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   4 	 1      $OUTDIR/test.cam  0.010 2010/01/01 04:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   5 	 2      $OUTDIR/test.cam  0.000 2010/01/01 05:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   6 	 2      $OUTDIR/test.cam -0.025 2010/01/01 06:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   7 	 2      $OUTDIR/test.cam  0.025 2010/01/01 07:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   8 	 2      $OUTDIR/test.cam  0.010 2010/01/01 08:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+   9 	 3      $OUTDIR/test.cam  0.000 2010/01/01 09:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  10 	 3      $OUTDIR/test.cam -0.025 2010/01/01 10:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  11 	 3      $OUTDIR/test.cam  0.025 2010/01/01 11:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  12 	 3      $OUTDIR/test.cam  0.010 2010/01/01 12:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  g       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  13 	 4      $OUTDIR/test.cam -0.025 2010/01/02 01:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  14 	 4      $OUTDIR/test.cam  0.025 2010/01/02 02:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  15 	 4      $OUTDIR/test.cam  0.010 2010/01/02 03:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  16 	 4      $OUTDIR/test.cam  0.000 2010/01/02 04:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  17 	 5      $OUTDIR/test.cam -0.025 2010/01/02 05:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  18 	 5      $OUTDIR/test.cam  0.025 2010/01/02 06:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  19 	 5      $OUTDIR/test.cam  0.010 2010/01/02 07:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  20 	 5      $OUTDIR/test.cam  0.000 2010/01/02 08:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  21 	 6      $OUTDIR/test.cam -0.025 2010/01/02 09:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  22 	 6      $OUTDIR/test.cam  0.025 2010/01/02 10:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  23 	 6      $OUTDIR/test.cam  0.010 2010/01/02 11:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+  24 	 6      $OUTDIR/test.cam -0.020 2010/01/02 12:00:00  $OUTDIR/$TESS_ID.$SKYCELL.wrp  r       $OUTDIR/$TESS_ID.$SKYCELL.diff
+end
+
+list stackdata
+# stk_id sky_id filename                      M_off filter  forcedgalaxy          
+  1      1 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.010 g       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+  2      2 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.020 g       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+  3      3 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.030 g       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+  4      1 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.025 r       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+  5      2 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.035 r       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+  6      3 	$OUTDIR/$TESS_ID.$SKYCELL.stk 0.045 r       $OUTDIR/$TESS_ID.$SKYCELL.fgal 
+end
+ 
+macro mkfull
+  init.db
+
+  
+
+  mkcatdir.cam $CMFFORMAT          $CATFORMAT
+  mkcatdir.stk $CMFFORMAT          $CATFORMAT
+  mkcatdir.wrp $CMFFORMAT\_Lensing $CATFORMAT
+  mkcatdir.dif $DIFFORMAT          $CATFORMAT
+  insert.stack.set
+
+  exec rm -rf testdata/catdir.mrg
+
+  # if CATFORMAT is specified, dvomerge will default to the .ptolemyrc version
+  exec dvomerge -matched-tables testdata/catdir.cam into testdata/catdir.mrg -D CATFORMAT $CATFORMAT
+  exec dvomerge -matched-tables testdata/catdir.stk into testdata/catdir.mrg
+  exec dvomerge -matched-tables testdata/catdir.wrp into testdata/catdir.mrg
+
+  # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits 
+  # exec relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1
+  echo relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1
+  exec relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits -cloud-limit 0.1
+end
+
+macro init.db
+
+  $dbhost = localhost
+  $dbname = gpc1test
+  $dbuser = dvo
+  $dbpass = dvo
+
+  $gscfile = `gconfig GSCFILE`
+  if ("$gscfile" == "not found")
+    echo "problem with dvo / ptolemy configuration.  check ~/.ptolemyrc"
+    break
+  end
+   
+  exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass
+  exec mkgpc1.sh create $dbhost $dbname $dbuser $dbpass
+
+  dbconnect $dbhost $dbuser $dbname -p $dbpass
+
+  # system-wide info
+  dbinsert ippToPspsFake (my_id) values (2)
+  dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, "$TESS_ID", "$SKYCELL")
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.cam
+  if ($0 != 3)
+    echo "mkcatdir.cam (cmftype) (dvotype)"
+    break
+  end
+
+  # tapPLAN 4
+
+  mkdir $OUTDIR
+
+  local i catdir ID stkID rawfile cmffile offset
+  $catdir = $OUTDIR/catdir.cam
+  exec rm -rf $catdir
+
+  $TIMEFORMAT = mjd
+  $TIMEREF = 2001/01/01,00:00:00
+
+  for i 0 $imagedata:n
+    list word -split $imagedata:$i
+    $ID = $word:0
+    $stkID = $word:1
+    sprintf rawfile "%s.%02d.txt" $word:2 $ID
+    sprintf cmffile "%s.%02d.smf" $word:2 $ID
+    $offset = $word:3
+    sprintf myDATE "%s" $word:4
+    sprintf myTIME "%s" $word:5
+    $filter = $word:7
+
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    echo ctimes -abs $myDATE,$myTIME -var mjd
+    ctimes -abs $myDATE,$myTIME -var mjd
+
+    # XXX for a simple test of ippToPsps, i need to generate smf files
+    # with some correspondence to gpc1 exposure smfs
+
+    # this means: 
+    # 1) a PHU with some basic header data
+    # 2) add extensions with EXTNAME of XYnn
+
+    # NOTE: mkcmf does not populate a PHU, so we have to fake it
+
+    # create an empty header and populate with the desired keywords
+    mcreate dummy 0 0
+    keyword dummy MJD-OBS  -wf $mjd
+    keyword dummy FILTERID -w  $filter.00000
+    keyword dummy EXPTIME  -wf 1.0
+
+    keyword dummy ZPT_ERR  -wf 0.01
+    keyword dummy EXPREQ   -wf 1.0
+    keyword dummy AIRMASS  -wf 1.0
+    keyword dummy RA       -wf $RA 
+    keyword dummy DEC	   -wf $DEC
+    keyword dummy CTYPE1   -w "RA---DIS"
+    keyword dummy CTYPE2   -w "DEC--DIS"
+    keyword dummy CRVAL1   -wf $RA
+    keyword dummy CRVAL2   -wf $DEC
+    keyword dummy CRPIX1   -wf 0.0
+    keyword dummy CRPIX2   -wf 0.0
+    keyword dummy CDELT1   -wf {1.0/3600}
+    keyword dummy CDELT2   -wf {1.0/3600}
+    keyword dummy PC001001 -wf 1.0
+    keyword dummy PC001002 -wf 0.0
+    keyword dummy PC002001 -wf 0.0
+    keyword dummy PC002002 -wf 1.0
+    keyword dummy NPLYTERM -wd 0
+    keyword dummy PCA1X3Y0 -wf 0.0
+    keyword dummy PCA1X2Y1 -wf 0.0
+    keyword dummy PCA1X1Y2 -wf 0.0
+    keyword dummy PCA1X0Y3 -wf 0.0
+    keyword dummy PCA1X2Y0 -wf 0.0
+    keyword dummy PCA1X1Y1 -wf 0.0
+    keyword dummy PCA1X0Y2 -wf 0.0
+    keyword dummy PCA2X3Y0 -wf 0.0
+    keyword dummy PCA2X2Y1 -wf 0.0
+    keyword dummy PCA2X1Y2 -wf 0.0
+    keyword dummy PCA2X0Y3 -wf 0.0
+    keyword dummy PCA2X2Y0 -wf 0.0
+    keyword dummy PCA2X1Y1 -wf 0.0
+    keyword dummy PCA2X0Y2 -wf 0.0
+    wd dummy $cmffile
+
+    mkinput $offset $rawfile
+    echo mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords
+    exec mkcmf -photcode SIMTEST.$filter.Chip $ADDNOISE -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 0 -coords
+
+    if ($i == 3)
+      echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry"
+      continue
+    end
+
+    if (0 && ($i == 5))
+      echo "NOTE: adding image $i 2x to DVO, but without detections : this simulates a failure in dvo contruction"
+
+      echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images
+      exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -image -dup-images
+    end
+
+    if (0 && ($i == 7))
+      echo "NOTE: adding detections from image $i 2x to DVO, but with wrong photcode and duplicate image ID : this simulates a failure in dvo construction"
+      echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J
+      exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -dup-images -image-id-override 7 -photcode 2MASS_J
+    end
+
+    echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
+    exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
+  end
+
+  if ($SMALLTEST)
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update
+
+    echo relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region 8 12 18 22 -update -apply-offsets
+    exec relastro -v -D CATDIR $catdir -D CAMERA simtest -images -update-chips -update-all-cameras -region 8 12 18 22 -update -apply-offsets
+  else
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+  end
+
+  if ($PARALLEL)
+    $hostname = `hostname`
+    exec rm -f $catdir/HostTable.dat
+    output $catdir/HostTable.dat
+    echo "# ID Hostname Catdir"
+    echo " 1 $hostname $catdir.p1"
+    echo " 2 $hostname $catdir.p2"
+    echo " 3 $hostname $catdir.p3"
+    output stdout
+    exec dvodist -out $catdir
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.stk
+  if ($0 != 3)
+    echo "mkcatdir.stk (cmftype) (dvotype)"
+    break
+  end
+
+  # tapPLAN 4
+
+  mkdir $OUTDIR
+
+  local i catdir stkID skyID rawfile cmffile offset
+  $catdir = $OUTDIR/catdir.stk
+  exec rm -rf $catdir
+
+  for i 0 $stackdata:n
+    list word -split $stackdata:$i
+    $stkID = $word:0
+    $skyID = $word:1
+    sprintf rawfile "%s.%02d.txt" $word:2 $stkID
+    sprintf cmffile "%s.%02d.cmf" $word:2 $stkID
+    $offset = $word:3
+    $filter = $word:4
+
+    ## this is not generating photcodes recognized as stack by relphot
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    mkinput $offset $rawfile
+    echo mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords -aper-offset 0.15
+    exec mkcmf -stack -photcode SIMTEST.$filter.SkyChip $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords -aper-offset 0.15
+
+    # add some required header keywords:
+    # exec echo "HIERARCH FPA.ZP =          {25.+$offset} / Magnitude zero point" > tmp.hdr
+    # exec fits_insert $cmffile tmp.hdr
+
+    echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER
+    exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER
+  end
+
+  if ($SMALLTEST)
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update
+  else
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+  end
+
+  if ($PARALLEL)
+    $hostname = `hostname`
+    exec rm -f $catdir/HostTable.dat
+    output $catdir/HostTable.dat
+    echo "# ID Hostname Catdir"
+    echo " 1 $hostname $catdir.p1"
+    echo " 2 $hostname $catdir.p2"
+    echo " 3 $hostname $catdir.p3"
+    output stdout
+    exec dvodist -out $catdir
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.wrp
+  if ($0 != 3)
+    echo "mkcatdir.wrp (cmftype) (dvotype)"
+    break
+  end
+
+  # tapPLAN 4
+
+  mkdir $OUTDIR
+
+  local i catdir ID stkID rawfile cmffile offset
+  $catdir = $OUTDIR/catdir.wrp
+  exec rm -rf $catdir
+
+  $TIMEFORMAT = mjd
+  $TIMEREF = 2001/01/01,00:00:00
+
+  for i 0 $imagedata:n
+    list word -split $imagedata:$i
+    $ID = $word:0
+    $stkID = $word:1
+    sprintf rawfile "%s.%02d.txt" $word:2 $ID; # use the cam-stage rawfile
+    sprintf cmffile "%s.%02d.smf" $word:6 $ID
+    $offset = $word:3
+    sprintf myDATE "%s" $word:4
+    sprintf myTIME "%s" $word:5
+    $filter = $word:7
+
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    echo ctimes -abs $myDATE,$myTIME -var mjd
+    ctimes -abs $myDATE,$myTIME -var mjd
+
+    ## this is not generating photcodes recognized as stack by relphot
+
+    mkinput $offset $rawfile
+    echo mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+    exec mkcmf -forcedwarp -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 2 -tess_id $TESS_ID -skycell $SKYCELL -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+
+    echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -xrad
+    exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass -xrad
+  end
+  
+  # add the forcedgalaxy files for the above warps:
+  for i 0 $stackdata:n
+    list word -split $stackdata:$i
+    $stkID = $word:0
+    sprintf rawfile "%s.%02d.txt" $word:2 $stkID
+    sprintf cmffile "%s.%02d.smf" $word:5 $stkID
+    $offset = $word:3
+    $filter = $word:4
+
+    ## this is not generating photcodes recognized as stack by relphot
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    echo mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords
+    exec mkcmf -forcedgalaxy -photcode SIMTEST.$filter.ForcedWarp $ADDNOISE $rawfile $cmffile -extroot SkyChip -date 2008/1/1 -time $i\:00:00 -mjd 54466.0 -radec $RA $DEC -type $1 -imageID $stkID -sourceID 1 -tess_id $TESS_ID -skycell $SKYCELL -coords
+
+    echo loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile 
+    exec loadgalphot -D CATDIR $catdir -D CAMERA simtest -photcode SIMTEST.$filter.ForcedWarp -image-id $stkID $cmffile
+  end
+
+  if ($SMALLTEST)
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -averages -update
+  else
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+  end
+
+  echo dvolens -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -update-objects -update
+  exec dvolens -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -update-objects -update
+
+  if ($PARALLEL)
+    $hostname = `hostname`
+    exec rm -f $catdir/HostTable.dat
+    output $catdir/HostTable.dat
+    echo "# ID Hostname Catdir"
+    echo " 1 $hostname $catdir.p1"
+    echo " 2 $hostname $catdir.p2"
+    echo " 3 $hostname $catdir.p3"
+    output stdout
+    exec dvodist -out $catdir
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+macro insert.exp
+  if ($0 != 4)
+    echo "USAGE: insert.exp (ID) (filename) (filter)"
+    break
+  end
+
+  local myID filename filter
+  $myID = $1
+  $filename = $2
+  $filter = $3
+
+  # a single exposure
+  echo $myDATE
+  echo $myTIME 
+
+  dbinsert rawExp (exp_id, exp_name, exp_time, filter, bg, bg_stdev, ra, decl, dateobs) values ($myID, "test.$myID", 10.0, "$filter.00000", 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, '$myDATE $myTIME')
+  dbinsert chipRun (exp_id, chip_id) values ($myID, $myID)
+  dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($myID, $myID, "testgroup", "38000M")
+  dbinsert camProcessedExp (cam_id, path_base, fault) values ($myID, "$filename", 0)
+  dbinsert fakeRun (fake_id, cam_id) values ($myID, $myID)
+  dbinsert warpRun (warp_id, fake_id, tess_id, state) values ($myID, $myID, "$TESS_ID", "full")
+  dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($myID, "$SKYCELL", "XY00")
+  dbinsert warpImfile (warp_id, skycell_id, warp_skyfile_id) values ($myID, "$SKYCELL", $myID)
+
+  dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($myID, "cam", "full", $myID, "catdir.cam")
+  dbinsert addProcessedExp (add_id, fault) values ($myID, 0)
+end
+
+macro insert.wrp
+  if ($0 != 4)
+    echo "USAGE: insert.wrp (warpID) (stackID) (filename)"
+    break
+  end
+
+  local wrpID stkID filename
+  $wrpID = $1
+  $stkID = $2
+  $filename = $3
+
+  # a single exposure
+  echo $myDATE
+  echo $myTIME 
+
+  dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($wrpID, "fullforce", "full", $stkID, $wrpID, "catdir.wrp")
+
+  # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids
+  dbinsert fullForceInput (ff_id, warp_id) values ($stkID, $wrpID)
+  dbinsert fullForceResult (ff_id, warp_id, software_ver, path_base) values ($stkID, $wrpID, "38000M", "$filename")
+end
+
+# this inserts a single stack, adding the exposures associated with this stack
+macro insert.stack
+  if ($0 != 5)
+    echo "USAGE: insert.stack (stkID) (skyID) (stkfile) (filter)"
+    break
+  end
+
+  local i stkID skyID expID expfile wrpfile stkfile filter
+  
+  $stkID   = $1
+  $skyID   = $2
+  $stkfile = $3
+  $filter  = $4
+
+  $skycalID = $stkID + 20
+
+  dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "$SKYCELL", "$filter.00000", "37500")
+
+  # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids
+  dbinsert fullForceRun (ff_id, skycal_id) values ($stkID, $skycalID)
+
+  $TIMEFORMAT = mjd
+  $TIMEREF = 2001/01/01,00:00:00
+
+  $mjdsum = 0.0
+  $Nstk = 0
+
+  for i 0 $imagedata:n
+    echo $imagedata:$i
+    list tmp -split $imagedata:$i
+    if ($stkID != $tmp:1) continue
+
+    $expID = $tmp:0
+    sprintf myDATE "%s" $tmp:4
+    sprintf myTIME "%s" $tmp:5
+
+    $expfile = $tmp:2
+    $myFilter  = $tmp:7
+    insert.exp $expID $expfile $myFilter
+    if ($myFilter != $filter)
+      echo "*** WARNING: filter mis-match between exp and stk: $myFilter vs $filter"
+      break
+    end
+
+    $wrpfile = $tmp:6
+    insert.wrp $expID $stkID $wrpfile
+
+    dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID)
+
+    echo ctimes -abs $myDATE,$myTIME -var mjd
+         ctimes -abs $myDATE,$myTIME -var mjd
+    $mjdsum += $mjd
+    $Nstk ++
+  end
+  $mjd = $mjdsum / $Nstk
+
+  dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd)
+  dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($skyID, $skycalID, $stkID)
+  dbinsert skycalResult (skycal_id, path_base) values ($skycalID, "$stkfile")
+  
+  dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($stkID, "skycal", "full", $skycalID, "catdir.stk")
+end
+
+macro mkcatdir.dif
+  if ($0 != 3)
+      echo "mkcatdir.cam (cmftype) (dvotype)"
+      break
+  end
+
+  # tapPLAN 4
+
+  mkdir $OUTDIR
+
+  local i catdir ID stkID rawfile cmffile offset
+  $catdir = $OUTDIR/catdir.dif
+  exec rm -rf $catdir
+
+  $TIMEFORMAT = mjd
+  $TIMEREF = 2001/01/01,00:00:00
+
+  for i 0 $imagedata:n
+    list word -split $imagedata:$i
+    $ID = $word:0
+    $stkID = $word:1
+    sprintf rawfile "%s.%02d.txt" $word:2 $stkID
+    sprintf cmffile "%s.%02d.smf" $word:8 $stkID
+    sprintf invfile "%s.%02d.inv.smf" $word:8 $stkID
+    $offset = $word:3
+    sprintf myDATE "%s" $word:4
+    sprintf myTIME "%s" $word:5
+    $filter = $word:7
+
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    echo ctimes -abs $myDATE,$myTIME -var mjd
+    ctimes -abs $myDATE,$myTIME -var mjd
+
+    mkinput $offset $rawfile
+    $diff_skyfile_id = $ID + 25
+    echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+    exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+
+    echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+    exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip $ADDNOISE -extroot SkyChip $rawfile $invfile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $diff_skyfile_id -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC -coords
+
+    insert.dif $ID $ID $stkID $word:8
+
+    if ($i == 3)
+      echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry"
+      continue
+    end
+
+    exec rm -rf addstar.txt
+    output addstar.txt
+    echo $cmffile
+    echo $invfile
+    output stdout
+    echo addstar -diff-inv -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest -list addstar.txt -D CATFORMAT $2 -quick-airmass
+    exec addstar -diff-inv -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest -list addstar.txt -D CATFORMAT $2 -quick-airmass
+  end
+  if ($SMALLTEST)
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN 
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -nloop 0 -D STAR_TOOFEW 0 -update -boundary-tree tess.3pi.fits -is-diff-db -statmode WT_MEAN
+  else
+    echo relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+    exec relphot -D CATDIR $catdir -D CAMERA simtest -region 8 12 18 22 -images g,r,i,z,y -update
+  end
+
+  if ($PARALLEL)
+    $hostname = `hostname`
+    exec rm -f $catdir/HostTable.dat
+    output $catdir/HostTable.dat
+    echo "# ID Hostname Catdir"
+    echo " 1 $hostname $catdir.p1"
+    echo " 2 $hostname $catdir.p2"
+    echo " 3 $hostname $catdir.p3"
+    output stdout
+    exec dvodist -out $catdir
+  end
+
+  # for i 0 $offset:n
+  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
+  # end
+
+  # tapDONE
+end
+
+
+macro insert.stack.set
+
+  local i stkID skyID cmffile filter
+
+  for i 0 $stackdata:n
+    list word -split $stackdata:$i
+    $stkID = $word:0
+    $skyID = $word:1
+    sprintf cmffile "%s.%02d.cmf" $word:2 $stkID
+    $filter = $word:4
+
+    if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
+
+    insert.stack $stkID $skyID $cmffile $filter
+  end
+
+  # I am setting these databases in a very fake way, with same in and out names!
+
+  # catdirs for the exposures
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.cam", 1, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0)
+  
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "catdir.cam")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0)
+
+  # catdirs for the stack
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.stk", 2, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (2, 0)
+  
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (2, 2, "full", "catdir.stk")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (2, 0)
+
+  # catdirs for the forced warps
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.wrp", 3, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (3, 0)
+  
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (3, 3, "full", "catdir.wrp")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (3, 0)
+
+  # catdirs for the diffs
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.dif", 4, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (4, 0)
+
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (4, 4, "full", "catdir.dif")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (4, 0)
+
+end
+
+macro insert.dif
+  if ($0 != 5)
+      echo "USAGE: insert.dif (diffID) (warpID) (stackID) (filename)"
+      break
+  end
+
+  local difID wrpID stkID filename
+  $difID = $1
+  $wrpID = $2
+  $stkID = $3
+  $filename = $4
+
+  echo $myDATE
+  echo $myTIME
+
+  $addID = $difID + 100
+  $diffSkyfileID = $difID + 25;
+  dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($addID, "diff", "full", $difID, $diffSkyfileID, "catdir.dif")
+  dbinsert addProcessedExp (add_id, fault) values ($addID, 0)
+
+  dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, "$TESS_ID", "full", 2, "38000M")
+  dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, "$TESS_ID", "$SKYCELL" )
+  dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, "$SKYCELL", "$filename")
+end
+
+# make a simple input file for mkcmf
+macro mkinput
+  if ($0 != 3)
+    echo "mkinput (offset) (output)"
+    break
+  end
+
+  local output 
+  $output = $2
+
+  exec rm -f $output
+
+  local i j ra dec
+  output $output
+  for i 10 1024 100
+    for j 10 1024 100
+      $ra  = 10.0 + $i*0.25/3600
+      $dec = 20.0 + $j*0.25/3600
+      fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f  %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+    end
+  end
+  # XXX add 2 duplicate detections
+  $i = 110; $j = 210
+  # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f  %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+  # fprintf " %6.1f %6.1f  %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+  $i = 210; $j = 110
+  # fprintf " %10.6f %10.6f %7.3f %6.1f %6.1f  %7.3f" $ra $dec {10.0 + $1 + 2.5*($i + $j)/1000.0} $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+  # fprintf " %6.1f %6.1f  %7.3f" $i $j {-15.0 + $1 + 2.5*($i + $j)/1000.0}
+  output stdout
+end
+
+macro check.mags
+
+  # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements
+  # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits 
+  # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits 
+
+  catdir testdata/catdir.mrg
+  skyregion 8 12 18 22
+  avextract ra dec g:psf:chp g:kron:chp g:psf:stk g:kron:stk g:psf:wrp g:kron:wrp
+
+  set dg:psf:chp:stk = g:psf:chp - g:psf:stk
+  set dg:psf:chp:wrp = g:psf:chp - g:psf:wrp
+
+  set dg:kron:chp:stk = g:kron:chp - g:kron:stk
+  set dg:kron:chp:wrp = g:kron:chp - g:kron:wrp
+
+  set dg:psf:kron:chp = g:psf:chp - g:kron:chp
+  set dg:psf:kron:stk = g:psf:stk - g:kron:stk
+  set dg:psf:kron:wrp = g:psf:wrp - g:kron:wrp
+
+  lim -n 0 g:psf:chp -0.2 0.2; clear; box; plot g:psf:chp dg:psf:chp:stk -c blue -pt 7; plot g:psf:chp dg:psf:chp:wrp -c red -pt 2
+
+  lim -n 1 g:kron:chp -0.2 0.2; clear; box; plot g:kron:chp dg:kron:chp:stk -c blue -pt 7; plot g:kron:chp dg:kron:chp:wrp -c red -pt 2
+
+  lim -n 2 g:psf:chp -0.2 0.2; clear; box; plot g:psf:chp dg:psf:kron:chp -c blue -pt 7; plot g:psf:chp dg:psf:kron:stk -c red -pt 2; plot g:psf:chp dg:psf:kron:wrp -c green -pt 2
+
+# for i 0 10 
+#   echo ra[$i] dec[$i] g:psf:chp[$i] g:kron:chp[$i] g:psf:stk[$i] g:psf:wrp[$i]
+# end
+end
+
+macro check.mags.meas
+
+  # NOTE: relphot refuses to set the warp averages if there is no boundary tree: otherwise it cannot choose the correct subset of measurements
+  # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -averages -boundary-tree tess.3pi.fits 
+  # relphot -D CATDIR testdata/catdir.mrg -D CAMERA simtest -region 8 12 18 22 -update -vv -images g,r -reset -reset-zpts -D STAR_TOOFEW 0 -boundary-tree tess.3pi.fits 
+
+  catdir testdata/catdir.mrg
+  skyregion 8 12 18 22
+
+  mextract ra dec mag:rel:psf mag:rel:kron mag:ave:psf mag:ave:kron photcode
+  set dM:psf = mag:ave:psf - mag:rel:psf
+  lim -n 0 mag:ave:psf dM:psf; clear; box; plot mag:ave:psf dM:psf
+  label -x "mag:ave:psf" -y "mag:ave:psf - mag:rel:psf"
+
+  set dM:kron = mag:ave:kron - mag:rel:kron
+  lim -n 1 mag:ave:kron dM:kron; clear; box; plot mag:ave:kron dM:kron
+  label -x "mag:ave:kron" -y "mag:ave:kron - mag:rel:kron"
+
+  set dM:psf:kron = mag:rel:psf - mag:rel:kron
+  lim -n 2 mag:rel:kron dM:psf:kron; clear; box; plot mag:rel:kron dM:psf:kron
+  label -x "mag:rel:kron" -y "mag:rel:psf - mag:rel:kron"
+end
+
+if ($SCRIPT)
+  if ($argv:n > 0)
+    if ("$argv:0" == "-parallel")
+      $PARALLEL = 1
+    end
+  end
+  mkfull
+  if (not($STATUS)) exit 1
+  exit 0
+end