Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery.sql
===================================================================
--- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery.sql	(revision 42593)
+++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery.sql	(revision 42594)
@@ -13,10 +13,18 @@
     JOIN skycell                USING (tess_id, skycell_id)
     -- join hook %s
+    -- for no-rerun:
+    -- LEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id
 WHERE 
-    %s.xcstackCamera.camera = 'test_cam1' AND
+    -- the first %s below should be filled with the xcamera database
+    -- the second should be the name of the camera corresponding to the
+    -- database in which the tool is run
+    %s.xcstackCamera.camera = '%s' AND
     %s.xccalRun.state = 'full' AND
     %s.xccalResult.quality = 0
-
+-- 'where' string is appended here
+    -- for no-rerun, also append:
+    -- AND xcff_id IS NULL
 
 -- this query is run in the database in which the original stacks were made (e.g., gpc1)
--- it refers to the database used to generate the xcstacks (e.g., xcamera)
+-- the external database (%s in front of the xcstackRun & xccalRun tables) refers to the database
+-- used to generate the xcstacks (e.g., xcamera)
Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c
===================================================================
--- /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c	(revision 42593)
+++ /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c	(revision 42594)
@@ -105,5 +105,5 @@
 }
 
-// NOTE: if I denormalize and populate (tess_id, skycell_id) in xcForceRun the  I do not need
+// NOTE: if I denormalize and populate (tess_id, skycell_id) in xcForceRun then I do not need
 // to join back to xcstackRun to get that info
 static bool definebyqueryMode(pxConfig *config)
@@ -131,6 +131,10 @@
     PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
 
-    // XXX is the needed or wanted?
+    // XXX is needed or wanted?
     // PXOPT_LOOKUP_STR(sources_path_base,   config->args, "-set_sources_path_base", false, false);
+
+    // first, we need to select the xccalRun(s) which need to have xcForceRuns.  we can limit the selection based on
+    // qualities of the xccalRun or the xcstackRun.  There will be one xcForceRun for each
+    // identified xccalRun.
 
     psMetadata *xccalWhereMD = psMetadataAlloc();
@@ -145,5 +149,4 @@
     }
 
-    // XXX do these need to specify stackRun?  can the filters be different between stackRun and xcstackRun?
     PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id",    "xcstackRun.skycell_id",      "LIKE");
     PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id",       "xcstackRun.tess_id",         "==");
@@ -155,8 +158,14 @@
     }
 
+    // once we have identified the xccalRun(s) for which we will generate the xcForceRuns, we
+    // need to select the input warps.  Normally, these would be all warps used to generate all
+    // stacks used to generate the xcstack skyfile associated with the xccalRun.  But we have
+    // the options to limit which warps we actually include.  
+
     psMetadata *warpWhereMD = psMetadataAlloc();
     pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label",      "warpRun.label",           "LIKE");
     pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group", "warpRun.data_group",      "LIKE");
     PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",         "warpRun.warp_id",         "==");
+    // XXX is is really necessary to limit the warps?
     if (!psListLength(warpWhereMD->list)) {
         psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required");
@@ -166,7 +175,4 @@
     }
     PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min",   "warpSkyfile.good_frac",  ">=");
-    // XXX do I need these or not?  the warp skyfiles need to match the stack skycells
-    PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",         "warpRun.tess_id",        "==");
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",          "rawExp.filter",          "LIKE");
 
     PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
@@ -186,4 +192,5 @@
     psStringAppend(&where, "\nAND %s", whereClause);
     psStringAppend(&select, " %s", where);
+
     psFree(whereClause);
     psFree(xccalWhereMD);
@@ -192,4 +199,9 @@
     if (!rerun) {
       psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id", xcamera);
+    }
+
+    psStringAppend(&joinHook, "\n %s, where);
+
+    if (
         psStringAppend(&joinHook, "\n %s\nAND xcForceRun.label = '%s'", where, label);
         psStringAppend(&select, "\nAND ff_id IS NULL");
Index: /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh
===================================================================
--- /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh	(revision 42594)
+++ /branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh	(revision 42594)
@@ -0,0 +1,236 @@
+#!/bin/csh
+
+which mariadb >& /dev/null
+@ has_mariadb = ($status == 0)
+
+which mysql >& /dev/null
+@ has_mysql = ($status == 0)
+
+set MYSQL = none
+if ($has_mariadb) then
+  set MYSQL = mariadb
+endif
+if (($has_mariadb == 0) && ($has_mysql)) then
+  set MYSQL = mysql
+endif
+
+echo "mysql is $MYSQL"
+
+set dbserver = localhost
+set dbuser = ipp
+set dbpass = ipp
+
+# create 3 databases: two cameras and a xcamera
+# also need to create ippadmin entries to match these camera databases:
+$MYSQL -h $dbserver -u $dbuser -p$dbpass ippadmin <<EOF
+  delete from projects where projname = 'test_cam1';
+  delete from projects where projname = 'test_cam2';
+  delete from projects where projname = 'test_xcam';
+  insert into projects (projname) values ('test_cam1');
+  insert into projects (projname) values ('test_cam2');
+  insert into projects (projname) values ('test_xcam');
+  drop database if exists test_cam1;
+  drop database if exists test_cam2;
+  drop database if exists test_xcam;
+  create database test_cam1;
+  create database test_cam2;
+  create database test_xcam;
+EOF
+
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam1 < make.xcstack.sql
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_cam2 < make.xcstack.sql
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam < make.xcstack.sql
+
+# generate skycell entry (needed for xcsky and xccal queries)
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_xcam <<EOF
+  insert into skycell (tess_id, skycell_id, radeg, decdeg, glong, glat, width, height) values ('Fake.Tess', 'skycell.2479.024', 200.412, 60.9857, 116.426, 55.7593, 0.433333, 0.438819);
+  insert into dbversion (schema_version) values ('0.1');
+EOF
+
+# insert data into the stack tables
+
+set mydir = `pwd`
+
+# I need to generate a more complex dataset.  I can use the same image (or use fake image names)
+# I will generate Nw warps, Ns stacks make from groups of Ni warps, but with some duplicates (ie, same warps used
+# in multiple stacks).  for additional complexity, the number of warps used in each stack will not be the same
+# for each stack (to mimic, e.g., a new stack made with additional new warps)
+
+rm -f test_cam1.sql
+
+# starting point for running indexes
+set warp_id = 1
+set stack_id = 1
+set nwarps_max = 5
+set nwarps_min = 3
+set cam = "cam1"
+
+set filters  = (g                r                g                )
+set skycells = (skycell.2479.024 skycell.2479.024 skycell.2479.025 )
+
+set ngroup = 1
+
+while ($ngroup <= $#filters)
+  set skycell = $skycells[$ngroup]
+  set filter  = $filters[$ngroup]
+  
+  @ warp_id_max = $warp_id + $nwarps_max - 1
+
+  # generate warps (two filters, two skycells)
+  while ($warp_id <= $warp_id_max) 
+    echo "insert into warpRun     (warp_id, state, workdir, label)          values ($warp_id, 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
+    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql
+    @ warp_id ++
+  end
+    
+  set nstack = 0
+
+  # generate stacks from nwarps_min inputs to nwarps_max input (e.g., 3, 4, 5, 6) 
+  while ($nstack < $nwarps_max - $nwarps_min + 1)
+    
+    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
+    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
+    
+    @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
+    @ warp_id_max = $warp_id_inp + $nstack + $nwarps_min - 1
+    while ($warp_id_inp <= $warp_id_max)
+      echo "insert into stackInputSkyfile (stack_id, warp_id) values ($stack_id, $warp_id_inp);" >> test_$cam.sql
+      @ warp_id_inp ++
+    end
+    @ nstack ++
+    @ stack_id ++
+  end
+
+  @ ngroup ++
+end
+
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql
+
+# starting point for running indexes
+set warp_id = 1
+set stack_id = 1
+set nwarps_max = 5
+set nwarps_min = 3
+set cam = "cam2"
+
+set filters  = (g                r                g                )
+set skycells = (skycell.2479.024 skycell.2479.024 skycell.2479.025 )
+
+set ngroup = 1
+
+while ($ngroup <= $#filters)
+  set skycell = $skycells[$ngroup]
+  set filter  = $filters[$ngroup]
+  
+  @ warp_id_max = $warp_id + $nwarps_max - 1
+
+  # generate warps (two filters, two skycells)
+  while ($warp_id <= $warp_id_max) 
+    echo "insert into warpRun     (warp_id, state, workdir, label)          values ($warp_id, 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
+    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, path_base) values ($warp_id, '$skycell', 'Fake.Tess', '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell');" >> test_$cam.sql
+    @ warp_id ++
+  end
+    
+  set nstack = 0
+
+  # generate stacks from nwarps_min inputs to nwarps_max input (e.g., 3, 4, 5, 6) 
+  while ($nstack < $nwarps_max - $nwarps_min + 1)
+    
+    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
+    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
+    
+    @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
+    @ warp_id_max = $warp_id_inp + $nstack + $nwarps_min - 1
+    while ($warp_id_inp <= $warp_id_max)
+      echo "insert into stackInputSkyfile (stack_id, warp_id) values ($stack_id, $warp_id_inp);" >> test_$cam.sql
+      @ warp_id_inp ++
+    end
+    @ nstack ++
+    @ stack_id ++
+  end
+
+  @ ngroup ++
+end
+
+$MYSQL -h $dbserver -u $dbuser -p$dbpass test_$cam < test_$cam.sql
+
+exit 2
+
+# define the xcameras:
+xcstacktool -dbname test_cam1 -addcamera -xcamera test_xcam
+xcstacktool -dbname test_cam2 -addcamera -xcamera test_xcam
+
+# define the xcameras:
+xcstacktool -dbname test_xcam -addcamera -xcamera test_cam1
+xcstacktool -dbname test_xcam -addcamera -xcamera test_cam2
+
+# define a xcstack:
+xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstack
+xcstacktool -dbname test_xcam -addbyquery    -xcamera test_cam2 -select_skycell_id skycell.2479.024 -select_instack_filter g -select_xcstack_filter g
+
+xcstacktool -dbname test_xcam -tosum > xcstack.md
+set stk_id = `grep xcstack_id xcstack.md | awk '{print $3}'`
+set workdir = `grep workdir xcstack.md | awk '{print $3}'`
+
+## run xcstack_skycell.pl or fake it
+if (0) then
+  # manually run xcstack : note the camera must match a real camera in ippconfig/system.conf
+  xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_id --outroot $workdir/test.xcs.v0 --run-state new --camera GPC1
+endif
+  # pretend to run xcstack
+  xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.stk.1 -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_id
+endif
+
+# if neither of the above are set, use pantasks to run xcstack
+xcskytool -dbname test_xcam -definebyquery -select_xcstack_id 1 -set_workdir $mydir/xcsky -set_label Test.xcsky
+
+xcskytool -dbname test_xcam -todo -label Test.xcsky
+
+xcskytool -dbname test_xcam -updaterun -label Test.xcsky -set_data_group foo -set_dist_group bar -set_note myNote
+
+xcskytool -dbname test_xcam -todo -label Test.xcsky > xcsky.md
+set sky_id = `grep xcsky_id xcsky.md | awk '{print $3}'`
+cat xcsky.md
+
+xcskytool -dbname test_xcam -inputs -xcsky_id $sky_id > xcsky.md
+set workdir = `grep workdir xcsky.md | awk '{print $3}'`
+cat xcsky.md
+
+## run xcsky.pl or fake it
+if (0) then
+  xcsky.pl --dbname test_xcam --xcsky_id $sky_id --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1
+else
+  xcskytool -dbname test_xcam -addresult -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 -hostname foobar -dtime_script 30 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_id
+endif
+
+xcskytool -dbname test_xcam -result -xcsky_id $sky_id
+
+if (0) then
+  xcskytool -dbname test_xcam -updateresult -xcsky_id 1 -fault 1
+  xcskytool -dbname test_xcam -updaterun -xcsky_id 1 -set_state new
+  xcskytool -dbname test_xcam -revert -xcsky_id 1
+endif
+
+# NOTE: xccalRun inherits the label, data_group, dist_group, reduction from xcskyRun
+xcskytool -dbname test_xcam -definexccalrun -select_xcsky_id 1
+
+xcskytool -dbname test_xcam -pendingxccalrun -label Test.xcsky > xccal.md
+set cal_id = `grep xccal_id xccal.md | awk '{print $3}'`
+cat xccal.md
+
+## run xccal.pl or fake it
+if (0) then
+  echo xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam
+       xccal.pl --xccal_id 1 --xcstack_id 1 --outroot $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 --path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.sky.1 --camera GPC1 --dbname test_xcam
+else
+  xcskytool -dbname test_xcam -addxccalresult -xccal_id 1 -path_base $workdir/Fake.Tess/skycell.2479.024/Fake.Tess.skycell.2479.024.cal.1 -hostname foobaz -zpt_obs 25.0
+endif
+
+xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_id
+
+if (0) then
+  xcskytool -dbname test_xcam -updatexccal    -xccal_id 1 -set_fault 1
+  xcskytool -dbname test_xcam -updatexccalrun -xccal_id 1 -set_state new
+  xcskytool -dbname test_xcam -revertxcal -xccal_id 1
+endif
+