Index: trunk/ippToPsps/test/mksasdata.dvo
===================================================================
--- trunk/ippToPsps/test/mksasdata.dvo	(revision 37246)
+++ trunk/ippToPsps/test/mksasdata.dvo	(revision 38974)
@@ -6,14 +6,24 @@
 if (not($?PARALLEL)) set PARALLEL = 0
 
+## this script must be run in this directory, output data goes to 'sasdata'
 $OUTDIR = sasdata
 
 if (not($?KEEPDVO)) set KEEPDVO = 0
 
-list expfiles
-  sasdata/o5745g0495o.356867.cm.982612
-  sasdata/o5745g0507o.356882.cm.982626
-  sasdata/o5745g0517o.356892.cm.982636
-  sasdata/o5745g0527o.356909.cm.982650
-end
+# some global value for ease of access
+# the RA,DEC (must?) match the actual skycell(s) in the test data
+$RA_FIELD = 333.8
+$DEC_FIELD = -0.2
+$TESS_ID = "RINGS.V3"
+
+$TIMEFORMAT = mjd
+$TIMEREF = 2001/01/01,00:00:00
+
+list expfiles -glob sasdata.v39/*.smf
+list stkfiles -glob sasdata.v39/*skycal*.cmf
+list wrpfiles -glob sasdata.v39/*ff*.cmf
+
+$catdirbase = catdir.sas
+$catdir = $OUTDIR/$catdirbase
 
 macro mkfull
@@ -21,7 +31,19 @@
   mkcatdir.cam
   mkcatdir.stk
+  mkcatdir.wrp
   mkcal
-  mkgpc1.cam
-  mkgpc1.stk
+
+  insert.cam
+  insert.wrp
+  insert.stk
+  insert.minis
+end
+
+macro mkdb
+  init.db
+  insert.cam
+  insert.wrp
+  insert.stk
+  insert.minis
 end
 
@@ -32,9 +54,4 @@
   $dbpass = dvo
 
-  # are these used to find the exposures in any way?
-  # center of the test skycell
-  $RA = 333.5
-  $DEC = -5.0
-
   break -auto off
   exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass
@@ -46,5 +63,30 @@
   # system-wide info
   dbinsert ippToPspsFake (my_id) values (2)
-  dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "skycell.1225.070")
+
+  $skycells:n = 0
+
+  for i 0 $stkfiles:n
+    rd im $stkfiles:$i -head
+    keyword im SKYCELL mySkyCell
+    $found = 0
+    for j 0 $skycells:n
+      if ($mySkyCell == $skycells:$j)
+        $found = 1
+        last
+      end
+    end
+    if ($found) continue
+    list skycells -add $mySkyCell
+
+    keyword im NAXIS1 Nx
+    keyword im NAXIS2 Ny
+    coords im -p {$Nx/2} {$Ny/2} 
+    dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, "$mySkyCell")
+  end
+    
+
+  # global catdir 
+  # mkdir $OUTDIR
+  # exec rm -rf $catdir
 end
 
@@ -55,19 +97,10 @@
     break
   end
-
-  mkdir $OUTDIR
-
-  local catdir
-  $catdirbase = catdir.sas.cam
-  $catdir = $OUTDIR/$catdirbase
-
-  exec rm -rf $catdir
 
   for i 0 $expfiles:n
     $expfile = $expfiles:$i
-
-    exec addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $expfile.smf -update
-  end
-  exec addstar -D CATDIR $catdir -region 330 340 -10 0 -resort
+    echo addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $expfile -update
+    exec addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $expfile -update
+  end
 end
 
@@ -75,113 +108,159 @@
 macro mkcatdir.stk
   if ($0 != 1)
-    echo "mkcatdir"
-    break
-  end
-
-  mkdir $OUTDIR
-
-  local catdir
-  $catdirbase = catdir.sas.stk
-  $catdir = $OUTDIR/$catdirbase
-  exec rm -rf $catdir
+    echo "mkcatdir.stk"
+    break
+  end
 
   # a single stack
-  $stkfile = sasdata/RINGS.V3.skycell.1225.070.stk.3366216.skycal.1589960
-  exec addstar -D CAMERA gpc1 -D CATFORMAT PS1_V5 -D CATDIR $catdir $stkfile.cmf
+  for i 0 $stkfiles:n
+    $stkfile = $stkfiles:$i
+    echo addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $stkfile -update -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER
+    exec addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $stkfile -update -quick-airmass -D ZERO_POINT_OPTION CHIP_HEADER
+  end
+  # exec addstar -D CATDIR $catdir -region {$RA-5.0} {$RA+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -resort
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro mkcatdir.wrp
+  if ($0 != 1)
+    echo "mkcatdir.wrp"
+    break
+  end
+
+  for i 0 $wrpfiles:n
+    $wrpfile = $wrpfiles:$i
+    echo addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $wrpfile -update -xrad -accept-astrom -quick-airmass
+    exec addstar -D CAMERA gpc1 -D CATDIR $catdir -D CATFORMAT PS1_V5 $wrpfile -update -xrad -accept-astrom -quick-airmass
+  end
+  # exec addstar -D CATDIR $catdir -region {$RA-5.0} {$RA+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -resort
 end
 
 macro mkcal
-
-  local catdir
-  $catdir = $OUTDIR/catdir.sas.cam
-
-  # merge the cam and stk databases together:
-  exec dvomerge $OUTDIR/catdir.sas.stk into $catdir
+  echo addstar -D CATDIR $catdir -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -resort
+  exec addstar -D CATDIR $catdir -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -resort
 
   # run relphot
-  echo relphot -D CATDIR $catdir -region 330 340 -10 0 -images g,r,i,z,y -update -boundary-tree tess.3pi.fits
-  exec relphot -D CATDIR $catdir -region 330 340 -10 0 -images g,r,i,z,y -update -boundary-tree tess.3pi.fits
-
-  echo relastro -v -D CATDIR $catdir -images -update-chips -region 330 340 -10 0 -update
-  exec relastro -v -D CATDIR $catdir -images -update-chips -region 330 340 -10 0 -update
-end
-
-# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
-macro mkgpc1.cam
-  if ($0 != 1)
-    echo "mkgpc1.cam"
-    break
-  end
-
-  local catdirbase
-  $catdirbase = catdir.sas.cam
+  # echo relphot -D CATDIR $catdir -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -images g,r,i,z,y -update -boundary-tree tess.3pi.fits
+  echo relphot -D CATDIR $catdir -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -averages -update -boundary-tree tess.3pi.fits
+  exec relphot -D CATDIR $catdir -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -averages -update -boundary-tree tess.3pi.fits
+
+  echo relastro -v -D CATDIR $catdir -images -update-chips -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -update
+  exec relastro -v -D CATDIR $catdir -images -update-chips -region {$RA_FIELD-5.0} {$RA_FIELD+5.0} {$DEC_FIELD-5.0} {$DEC_FIELD+5.0} -update
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro insert.cam
+  if ($0 != 1)
+    echo "USAGE: insert.cam"
+    break
+  end
+
+  local i filename path_base expname expfile
 
   for i 0 $expfiles:n
-    $expfile = $expfiles:$i
-
-    # are these used to find the exposures in any way?
-    $RA = 333.9
-    $DEC = -3.8
+    
+    $filename = $expfiles:$i
+    strlen $filename Nchar
+    substr $filename 0 {$Nchar - 4} path_base; # the database needs the path without the extension (.smf/.cmf)
 
     # a single exposure
     $expID = $i + 1
-    substr $expfile 8 11 expname
-
-    dbinsert rawExp (exp_id, exp_name, exp_time, bg, bg_stdev, ra, decl, dateobs) values ($expID, "$expname", 10.0, 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, "2010/01/01 00:00:00")
+
+    basename $filename -var expfile
+    substr $expfile 0 11 expname
+
+    $filterID = `echo $filename | fields FILTERID | prcol 2`
+    $mjdExp = `echo $filename | fields MJD-OBS | prcol 2`
+
+    ctimes -ref $mjdExp -var dateobs
+    strsub $dateobs , " " -var myDateObs
+
+    # many of these values are fakes for query purposes only.
+    dbinsert rawExp (exp_id, exp_name, exp_time, filter, bg, bg_stdev, ra, decl, dateobs) values ($expID, "$expname", 10.0, "$filterID", 100.0, 2.0, {$RA*3.1415/180.0}, {$DEC*3.1415/180.0}, "$myDateObs")
     dbinsert chipRun (exp_id, chip_id) values ($expID, $expID)
     dbinsert camRun (chip_id, cam_id, dist_group, software_ver) values ($expID, $expID, "testgroup", "38000M")
-    dbinsert camProcessedExp (cam_id, path_base, fault) values ($expID, "$expfile", 0)
+    dbinsert camProcessedExp (cam_id, path_base, fault) values ($expID, "$path_base", 0)
     dbinsert fakeRun (fake_id, cam_id) values ($expID, $expID)
-    dbinsert warpRun (warp_id, fake_id) values ($expID, $expID)
+    dbinsert warpRun (warp_id, fake_id, tess_id, state) values ($expID, $expID, $TESS_ID, "full")
+
+    # do I care if the class ID is invalid?
     dbinsert warpSkyCellMap (warp_id, skycell_id, class_id) values ($expID, "skycell.1225.070", "XY00")
     dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($expID, "cam", "full", $expID, "$catdirbase")
     dbinsert addProcessedExp (add_id, fault) values ($expID, 0)
-    
-    # catdirs for the exposures
-    dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 1, "merged")
-    dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0)
-    
-    dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "$catdirbase")
-    dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0)
-  end
-end
-
-# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
-macro mkgpc1.stk
-  if ($0 != 1)
-    echo "mkgpc1.stk"
-    break
-  end
-
-  local catdirbase
-  $catdirbase = catdir.sas.cam
-
-  $TIMEFORMAT = mjd
-  $TIMEREF = 2001/01/01,00:00:00
-  ctimes -abs 2010/01/01,00:00:00 -var mjd
-
-  # a single stack
-  $stkfile = sasdata/RINGS.V3.skycell.1225.070.stk.3366216.skycal.1589960
-
-  break -auto off
-  $tmpline = `echo $stkfile.cmf | fields IMAGEID`
-  list words -split $tmpline
-  $stkID = $words:1
-  break -auto on
-
-  $expID = 1
-  dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "skycell.1225.070", "i.00000", "37500M")
-  dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID)
-  dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd)
-  dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($stkID, $stkID, $stkID)
-  dbinsert skycalResult (skycal_id, path_base) values ($stkID, "$stkfile")
-  dbinsert addRun (stage, state, stage_id, minidvodb_name) values ("skycal", "full", $stkID, "$catdirbase")
-
-  # catdirs for the stack
-  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 2, "merged")
-  dbinsert minidvodbProcessed (minidvodb_id, fault) values (2, 0)
-  
-  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (2, 2, "full", "$catdirbase")
-  dbinsert mergedvodbProcessed (merge_id, fault) values (2, 0)
+  end
+end
+
+macro insert.wrp
+  if ($0 != 1)
+    echo "USAGE: insert.wrp"
+    break
+  end
+
+  local wrpID stkID filename path_base
+
+  $stkID = 1
+
+  for i 0 $wrpfiles:n
+    $filename = $wrpfiles:$i
+    strlen $filename Nchar
+    substr $filename 0 {$Nchar - 4} path_base; # the database needs the path without the extension (.smf/.cmf)
+
+    # a single exposure
+    $wrpID = $i + 1
+
+    # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids
+    dbinsert fullForceInput (ff_id, warp_id) values ($stkID, $wrpID)
+    dbinsert fullForceResult (ff_id, warp_id, software_ver, path_base) values ($stkID, $wrpID, "38000M", "$path_base")
+
+    dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($wrpID, "fullforce", "full", $stkID, $wrpID, "catdirbase")
+
+    if ($i > 10) last
+  end
+end
+
+# create a populated catdir with a set of cmf files, save the filenames in list for use by insert.exp
+macro insert.stk
+  if ($0 != 1)
+    echo "USAGE: insert.stk"
+    break
+  end
+
+  local stkID skycell filter filename path_base
+
+  for i 0 $stkfiles:n
+    $filename = $stkfiles:$i
+    strlen $filename Nchar
+    substr $filename 0 {$Nchar - 4} path_base; # the database needs the path without the extension (.smf/.cmf)
+
+    $stkID   = `echo $filename | fields IMAGEID | prcol 2`
+    $skycell = `echo $filename | fields SKYCELL | prcol 2`
+    $filter  = `echo $filename | fields FPA.FILTERID | prcol 2`
+    $mjd     = `echo $filename | fields MJD-OBS | prcol 2`
+    
+    # the warp_id is only needed to generate the list of input exposures
+    $expID = $i + 1
+    dbinsert stackRun (stack_id, skycell_id, filter, software_ver) values ($stkID, "$skycell", "$filter", "37500M")
+    
+    # we have a single fullForceRun for each stack, so we can match fullForceRun ids to stackRun ids
+    $skyID    = $stkID
+    $skycalID = $stkID + 20
+    dbinsert fullForceRun (ff_id, skycal_id) values ($stkID, $skycalID)
+    
+    dbinsert stackInputSkyfile (stack_id, warp_id) values ($stkID, $expID)
+    dbinsert stackSumSkyfile (stack_id, mjd_obs) values ($stkID, $mjd)
+    dbinsert skycalRun (sky_id, skycal_id, stack_id) values ($skyID, $skycalID, $stkID)
+    dbinsert skycalResult (skycal_id, path_base) values ($skycalID, "$path_base")
+    
+    dbinsert addRun (add_id, stage, state, stage_id, minidvodb_name) values ($stkID, "skycal", "full", $skycalID, "$catdirbase")
+  end
+end
+
+macro insert.minis
+  # catdirs for the exposures
+  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("$catdirbase", 1, "merged")
+  dbinsert minidvodbProcessed (minidvodb_id, fault) values (1, 0)
+    
+  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (1, 1, "full", "$catdirbase")
+  dbinsert mergedvodbProcessed (merge_id, fault) values (1, 0)
 end