Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh
===================================================================
--- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh	(revision 42602)
+++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh	(revision 42603)
@@ -101,5 +101,5 @@
     echo "insert into fakeRun     (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql
     echo "insert into warpRun     (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
-    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql
+    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/warp.$warp_id.$skycell', 'full');" >> test_$cam.sql
     @ warp_id ++
   end
@@ -110,6 +110,6 @@
   while ($nstack < $nwarps_max - $nwarps_min + 1)
     
-    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
-    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
+    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir/stacks.$cam', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
+    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/stacks.$cam/stack.$stack_id.$skycell', 0.95, 0, 0);" >> test_$cam.sql
     
     @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
@@ -157,5 +157,5 @@
     echo "insert into fakeRun     (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql
     echo "insert into warpRun     (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
-    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql
+    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/warp.$warp_id.$skycell', 'full');" >> test_$cam.sql
     @ warp_id ++
   end
@@ -166,6 +166,6 @@
   while ($nstack < $nwarps_max - $nwarps_min + 1)
     
-    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
-    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
+    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir/stacks.$cam', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
+    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/stacks.$cam/stack.$stack_id.$skycell', 0.95, 0, 0);" >> test_$cam.sql
     
     @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
@@ -203,5 +203,5 @@
 while ($nstack < $nwarps_max - $nwarps_min + 1)
   # define xcstacks by matching labels:
-  xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1
+  xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir/xcstack -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1
 
   # we have to define the relationship between instack and xcstack filters, so both must be specified
@@ -220,14 +220,21 @@
 set Tskycells = (`grep skycell_id xcstack.md | awk '{print $3}'`)
 set workdir = `grep workdir xcstack.md | awk '{print $3}' | head -n 1`
+mkdir -p $workdir
 
 # pretend to run xcstack
 set n = 1
 while ($n <= $#stk_ids)
-  xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n]
+  # OUTROOT is defined in xcstack.pro based on workdir : "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSTACK_ID
+  if (0 && ($n == 1)) then
+    # manually run xcstack : note the camera must match a real camera in ippconfig/system.conf
+    mkdir -p $workdir/Fake.Tess/$Tskycells[$n]
+    echo xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_ids[$n] --outroot $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] --run-state new --camera GPC1
+    xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_ids[$n] --outroot $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] --run-state new --camera GPC1
+  else
+    xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n]
+  endif
   @ n++
 end
 
-# exit 2
-
 echo "******** xcskytool steps (static sky) ********"
 
@@ -235,10 +242,10 @@
 
 # do the first set as separate filters
-xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r
-xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g
+xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r
+xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g
 
 # do the second and third sets as multi-filters
-xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r
-xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r
+xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r
+xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r
 
 xcskytool -dbname test_xcam -todo -label Test.xcsky.0
@@ -259,6 +266,13 @@
   set n = 1
   while ($n <= $#sky_ids)
-    xcskytool -dbname test_xcam -addresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n]
-    xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n]
+    # OUTROOT is defined in xcsky.pro based on workdir : "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKY_ID
+    if (0 && ($n == 2)) then
+      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
+      echo xcsky.pl --dbname test_xcam --xcsky_id $sky_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] --camera GPC1
+      xcsky.pl --dbname test_xcam --xcsky_id $sky_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] --camera GPC1
+    else
+      xcskytool -dbname test_xcam -addresult -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n]
+      xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n]
+    endif
     @ n++
   end
@@ -281,9 +295,9 @@
 # queue all three labels
 foreach label (0 1 2)
+  # workdir is inherited from xcsky if not specified here
   xcskytool -dbname test_xcam -definexccalrun -select_label Test.xcsky.$label -set_label Test.xccal.$label
 end 
 
 # mysql> select xccal_id, xcsky_id, xcstack_id, xccalRun.label as calLabel, xcskyRun.label as skyLabel, xcstackRun.label as stkLabel, xcstackRun.filter, xcskyResult.path_base, xcstackSumSkyfile.path_base from xccalRun join xcskyInput using (xcsky_id, xcstack_id) join xcskyRun using (xcsky_id) join xcskyResult using (xcsky_id) join xcstackRun using (xcstack_id) join xcstackSumSkyfile using (xcstack_id);
-
 # NOTE: xccalRun inherits the label, data_group, dist_group, reduction from xcskyRun
 
@@ -291,12 +305,21 @@
   xcskytool -dbname test_xcam -pendingxccalrun -label Test.xccal.$label > xccal.md
   set cal_ids   = (`grep xccal_id   xccal.md | awk '{print $3}'`)
+  set stk_ids   = (`grep xcstack_id xccal.md | awk '{print $3}'`)
   set Tskycells = (`grep skycell_id xccal.md | awk '{print $3}'`)
   set workdir   = (`grep workdir    xccal.md | awk '{print $3}'`)
+  set pathbase  = (`grep path_base  xccal.md | awk '{print $3}'`)
   
   ## pretend to run xcsky.pl
   set n = 1
   while ($n <= $#cal_ids)
-    xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n]
-    xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n]
+    # OUTROOT is defined in xccal.pro based on workdir : "%s/%s/%s/%s.%s.cal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKYCAL_ID  ****** TBD *****
+    if (0 && ($n == 1)) then
+      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
+      echo xccal.pl --dbname test_xcam --xccal_id $cal_ids[$n] --xcstack_id $stk_ids[$n] --path_base $pathbase[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] --camera GPC1
+           xccal.pl --dbname test_xcam --xccal_id $cal_ids[$n] --xcstack_id $stk_ids[$n] --path_base $pathbase[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] --camera GPC1
+    else
+      xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n]
+      xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n]
+    endif
     @ n++
   end
@@ -315,6 +338,6 @@
 
 foreach label (0 1 2)
-  xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam 
-  xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam 
+  xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir $mydir/xcff -xcamera test_xcam 
+  xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir $mydir/xcff -xcamera test_xcam 
 end
 
@@ -324,11 +347,21 @@
   set cam = cam1
   xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md
-  set xcff_ids   = (`grep xcff_id   xcff.todo.md | awk '{print $3}'`)
-  set warp_ids   = (`grep warp_id   xcff.todo.md | awk '{print $3}'`)
+  set xcff_ids   = (`grep xcff_id    	    xcff.todo.md | awk '{print $3}'`)
+  set warp_ids   = (`grep warp_id    	    xcff.todo.md | awk '{print $3}'`)
+  set Tskycells  = (`grep skycell_id 	    xcff.todo.md | awk '{print $3}'`)
+  set workdir    = (`grep workdir    	    xcff.todo.md | awk '{print $3}'`)
+  set warppaths  = (`grep warp_path_base    xcff.todo.md | awk '{print $3}'`)
+  set srcpaths   = (`grep sources_path_base xcff.todo.md | awk '{print $3}'`)
   
   ## pretend to run xcsky.pl
   set n = 1
   while ($n <= $#xcff_ids)
-    xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0
+    # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.wrp.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID ***** TBD *****
+    if (1 && ($cam == "cam1") && ($n == 1)) then
+      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
+      echo xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1
+           xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1
+    else
+    xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0
     xcfftool -dbname test_$cam -result    -xcff_id $xcff_ids[$n]
     @ n++
@@ -336,12 +369,29 @@
 
   # get a list of the unique xcff_ids 
-  set xcff_ids   = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`)
+  set xcff_ids_u   = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`)
 
   ## pretend to run xcsky.pl
   set n = 1
-  while ($n <= $#xcff_ids)
-    xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids[$n]
-    xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio
-    xcfftool -dbname test_$cam -summary    -xcff_id $xcff_ids[$n]
+  while ($n <= $#xcff_ids_u)
+    # get the workdir from above which matches this uniq xcff_ids:
+    set workdir_u = ""
+    set skycell_u = ""
+    set t = 1
+    while ($t <= $#xcff_ids)
+      if ($xcff_ids[$t] == $xcff_ids_u[$n]) then
+        set workdir_u = $workdir[$n]
+        set skycell_u = $Tskycells[$n]
+      endif
+      @ t++
+    end
+    if ($workdir_u == "") then
+      echo "ERROR: failed to find workdir for $xcff_ids_u[$n] from $workdir"
+      exit 2
+    endif
+
+    # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCFF_ID **** TBD ***
+    xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids_u[$n]
+    xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids_u[$n] -path_base $workdir_u/Fake.Tess/$skycell_u/Fake.Tess.$skycell_u.xcff.$xcff_ids_u[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio
+    xcfftool -dbname test_$cam -summary    -xcff_id $xcff_ids_u[$n]
     @ n++
   end