Changeset 42596

Timestamp:

Jan 19, 2024, 8:12:15 AM (2 years ago)

Author:

eugene

Message:

working on xcfftool

Location:

branches/eam_branches/ipp-20230313/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/xcfftool_definebyquery.sql (deleted)
• share/xcfftool_definebyquery_pt1.sql (added)
• share/xcfftool_definebyquery_pt2.sql (added)
• share/xcfftool_definebyquery_select_warps.sql (modified) (1 diff)
• share/xcfftool_result.sql (modified) (1 diff)
• share/xcfftool_revert.sql (modified) (1 diff)
• share/xcfftool_revertsummary.sql (modified) (1 diff)
• share/xcfftool_summary.sql (modified) (1 diff)
• share/xcfftool_toadvance.sql (modified) (1 diff)
• share/xcfftool_todo.sql (modified) (1 diff)
• share/xcfftool_updaterun.sql (modified) (1 diff)
• share/xcskytool_definebyquery_select.sql (modified) (1 diff)
• share/xcskytool_todo.sql (modified) (2 diffs)
• src/Makefile.am (modified) (3 diffs)
• src/xcfftool.c (modified) (35 diffs)
• src/xcfftool.h (modified) (1 diff)
• src/xcfftoolConfig.c (modified) (13 diffs)
• src/xcskytool.c (modified) (1 diff)

branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am

@@ -581 +581 @@
         xcskytool_result.sql \
         xcskytool_revert.sql \
-        xcskytool_revert_update.sql
+        xcskytool_revert_update.sql \
+        xcfftool_definebyquery_pt1.sql \
+        xcfftool_definebyquery_pt2.sql \
+        xcfftool_definebyquery_select_warps.sql \
+        xcfftool_result.sql \
+        xcfftool_revert.sql \
+        xcfftool_revertsummary.sql \
+        xcfftool_summaryinputs.sql \
+        xcfftool_summary.sql \
+        xcfftool_toadvance.sql \
+        xcfftool_todo.sql

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_select_warps.sql

@@ -1 +1 @@
 SELECT warp_id
-FROM warpRun join warpSkyfile using(warp_id, tess_id)
-    JOIN fakeRun using(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    JOIN skycell ON skycell.tess_id = warpSkyfile.tess_id AND skycell.skycell_id = warpSkyfile.skycell_id
-WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s'
-    AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0
+FROM stackInputSkyfile
+JOIN warpRun using (warp_id)
+JOIN warpSkyfile using(warp_id, tess_id)
+WHERE (stack_id = %lld)
+AND   warpRun.state = 'full'
+AND   warpSkyfile.quality = 0
+AND   warpSkyfile.fault = 0

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_result.sql

@@ -1 +1 @@
 SELECT
-    fullForceResult.*,
-    fullForceRun.state,
-    fullForceRun.label,
-    fullForceRun.data_group,
-    fullForceRun.xccal_id,
-    fullForceRun.workdir,
-    fullForceRun.sources_path_base,
+    xcForceResult.*,
+    xcForceRun.state,
+    xcForceRun.label,
+    xcForceRun.data_group,
+    xcForceRun.xccal_id,
+    xcForceRun.workdir,
+    xcForceRun.sources_path_base,
     CONCAT_WS('.', warp_id, skycell_id) AS dist_component,
-    stackRun.tess_id,
-    stackRun.skycell_id,
-    stackRun.filter,
+    xcForceRun.tess_id,
+    xcForceRun.skycell_id,
     warpImfile.warp_skyfile_id,
     rawExp.exp_id,
     rawExp.exp_name,
     rawExp.camera
-FROM fullForceRun
-    JOIN fullForceResult USING(ff_id)
-    JOIN xccalRun using(xccal_id)
-    JOIN stackRun using(stack_id)
+FROM xcForceRun
+    JOIN xcForceResult USING(xcff_id)
     JOIN warpRun using(warp_id, tess_id)
     JOIN warpSkyfile USING(warp_id, tess_id, skycell_id)

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revert.sql

@@ -1 @@
-DELETE FROM fullForceResult
-USING fullForceResult, fullForceRun, xccalRun, stackRun, skycell
-WHERE fullForceRun.ff_id = fullForceResult.ff_id
-    AND fullForceResult.fault != 0
-    AND fullForceRun.xccal_id = xccalRun.xccal_id
-    AND xccalRun.stack_id = stackRun.stack_id
-    AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id =  skycell.skycell_id
+DELETE FROM xcForceResult
+USING xcForceResult, xcForceRun
+WHERE xcForceRun.xcff_id = xcForceResult.xcff_id
+    AND xcForceResult.fault != 0

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revertsummary.sql

@@ -1 @@
-DELETE FROM fullForceSummary
-USING fullForceSummary, fullForceRun, xccalRun, stackRun, skycell
-WHERE fullForceRun.ff_id = fullForceSummary.ff_id
-    AND fullForceSummary.fault != 0
-    AND fullForceRun.xccal_id = xccalRun.xccal_id
-    AND xccalRun.stack_id = stackRun.stack_id
-    AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id =  skycell.skycell_id
+DELETE FROM xcForceSummary
+USING xcForceSummary, xcForceRun, skycell
+WHERE xcForceRun.xcff_id = xcForceSummary.xcff_id
+    AND xcForceSummary.fault != 0
+    AND xcForceRun.tess_id = skycell.tess_id AND xcForceRun.skycell_id =  skycell.skycell_id

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_summary.sql

@@ -1 +1 @@
 SELECT DISTINCT
-    fullForceSummary.*,
-    fullForceRun.state,
-    fullForceRun.label,
-    fullForceRun.data_group,
-    fullForceRun.xccal_id,
-    fullForceRun.workdir,
-    fullForceRun.sources_path_base,
+    xcForceSummary.*,
+    xcForceRun.state,
+    xcForceRun.label,
+    xcForceRun.data_group,
+    xcForceRun.xccal_id,
+    xcForceRun.workdir,
+    xcForceRun.sources_path_base,
+    xcForceRun.tess_id,
+    xcForceRun.skycell_id,
     'summary' AS dist_component,
-    stackRun.stack_id,
-    stackRun.tess_id,
-    stackRun.skycell_id,
-    stackRun.filter,
-    rawExp.camera
-FROM fullForceRun
-    JOIN fullForceSummary USING(ff_id)
-    JOIN fullForceInput USING(ff_id)
-    JOIN xccalRun using(xccal_id)
-    JOIN stackRun using(stack_id)
-    JOIN warpRun using(warp_id, tess_id)
-    JOIN warpSkyfile USING(warp_id, tess_id, skycell_id)
-    JOIN fakeRun using(fake_id)
-    JOIN camRun using(cam_id)
-    JOIN chipRun using(chip_id)
-    JOIN rawExp using(exp_id)
+FROM xcForceRun
+    JOIN xcForceSummary USING(xcff_id)
+    JOIN xcForceInput USING(xcff_id)
+    JOIN skycell USING (tess_id, skycell_id)

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_toadvance.sql

@@ -1 +1 @@
 SELECT
-    fullForceRun.*,
-    stackRun.tess_id,
-    stackRun.skycell_id,
-    stackRun.filter,
-    rawExp.camera
-FROM fullForceRun
-    JOIN fullForceInput USING(ff_id)
-    JOIN xccalRun USING(xccal_id)
-    JOIN stackRun USING(stack_id)
-    JOIN warpRun USING(warp_id, tess_id)
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    LEFT JOIN fullForceResult USING(ff_id, warp_id)
-    LEFT JOIN fullForceSummary USING(ff_id)
-WHERE fullForceRun.state = 'new' AND fullForceSummary.ff_id IS NULL
+    xcForceRun.xcff_id,
+    xcForceRun.tess_id,
+    xcForceRun.skycell_id,
+    xcForceRun.workdir,
+    xcForceRun.reduction
+FROM xcForceRun
+    JOIN xcForceInput USING(xcff_id)
+    LEFT JOIN xcForceResult USING(xcff_id, warp_id)
+    LEFT JOIN xcForceSummary USING(xcff_id)
+WHERE xcForceRun.state = 'new' AND xcForceSummary.ff_id IS NULL
     -- WHERE hook %s
-    GROUP BY ff_id
-    HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id)
-        AND SUM(fullForceResult.fault) = 0
+    GROUP BY xcff_id
+    HAVING COUNT(xcForceInput.warp_id) = COUNT(xcForceResult.warp_id)
+        AND SUM(xcForceResult.fault) = 0

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_todo.sql

@@ -1 +1 @@
 SELECT
-    fullForceRun.*,
-    fullForceInput.warp_id,
-    stackRun.tess_id,
-    stackRun.skycell_id,
-    stackRun.filter,
-    rawExp.camera,
+    xcForceRun.*,
+    xcForceInput.warp_id,
     warpSkyfile.path_base AS warp_path_base
-FROM fullForceRun
-    JOIN fullForceInput USING(ff_id)
-    JOIN xccalRun USING(xccal_id)
-    JOIN stackRun USING(stack_id)
+FROM xcForceRun
+    JOIN xcForceInput USING(xcff_id)
     JOIN warpRun USING(warp_id, tess_id)
     JOIN warpSkyfile USING(warp_id, tess_id, skycell_id)
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    LEFT JOIN fullForceResult USING(ff_id, warp_id)
-WHERE fullForceRun.state = 'new'
-    AND fullForceResult.ff_id IS NULL
+    LEFT JOIN xcForceResult USING(xcff_id, warp_id)
+WHERE xcForceRun.state = 'new'
+    AND xcForceResult.xcff_id IS NULL
     AND warpSkyfile.data_state = 'full'

branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql

@@ -1 +1 @@
 UPDATE xcForceRun
-JOIN %s.xccalRun            USING(xccal_id)
-JOIN %s.xccalResult         USING (xccal_id)
-JOIN %s.xcstackInputSkyfile USING (xcstack_id) -- exposes the stack_id
-JOIN %s.xcstackCamera       USING (camera_id) -- select the camera matching this db
-JOIN stackRun               USING (stack_id)
-JOIN skycell USING(tess_id, skycell_id)");
+JOIN skycell USING(tess_id, skycell_id);

branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql

@@ -22 +22 @@
 LEFT JOIN (
   SELECT DISTINCT xcsky_id,
-    xcstackRun.skycell_id,
-    xcstackRun.tess_id,
-    xcstackRun.data_group,
+    MIN(xcstackRun.skycell_id) as skycell_id,
+    MIN(xcstackRun.tess_id) as tess_id,
+    MIN(xcstackRun.data_group) as data_group,
     COUNT(xcstack_id) AS num_filter
   FROM xcskyRun

branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql

@@ +1 @@
+ -- we implicitly assume that all xcstackRuns used for a xcskyRun have the same tess_id and skycell_id 
 SELECT
     xcskyRun.xcsky_id,
@@ -5 +6 @@
     xcskyRun.label,
     xcskyRun.state,
-    MIN(xcstackRun.tess_id) as tess_id, -- we implicitly assume that all xcstackRuns used for a xcskyRun have the same tess_id
-    MIN(xcstackRun.skycell_id) as skycell_id, -- and skycell_id 
+    MIN(xcstackRun.tess_id) as tess_id,
+    MIN(xcstackRun.skycell_id) as skycell_id,
     TRUNCATE(MIN(skycell.radeg)/15., 4) as rahours,
     MIN(skycell.radeg) as radeg,

branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am

@@ -26 +26 @@
         xcstacktool \
         xcskytool \
+        xcfftool \
         warptool \
         receivetool \
@@ -86 +87 @@
         xcstacktool.h \
         xcskytool.h \
+        xcfftool.h \
         staticskytool.h \
         warptool.h \
@@ -261 +263 @@
     xcskytoolConfig.c
 
+xcfftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+xcfftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+xcfftool_SOURCES = \
+    xcfftool.c \
+    xcfftoolConfig.c
+
 pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la

branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c

@@ -33 +33 @@
 
 static bool definebyqueryMode(pxConfig *config);
-static bool defineforstacksMode(pxConfig *config);
 static bool updaterunMode(pxConfig *config);
 static bool todoMode(pxConfig *config);
@@ -48 +47 @@
 static bool importrunMode(pxConfig *config);
 
-static bool setxcForceRunState(pxConfig *config, psS64 sky_id, const char *state);
-static psS64 getCameraIDfromName(pxConfig *config, const char *camera);
+static bool setxcForceRunState(pxConfig *config, psS64 xcff_id, const char *state);
+static psS64 getCameraIDfromName(pxConfig *config, const char *dbname, const char *camera);
 
 # define MODECASE(caseName, func) \
@@ -70 +69 @@
     switch (config->mode) {
         MODECASE(XCFFTOOL_MODE_DEFINEBYQUERY,     definebyqueryMode);
-        MODECASE(XCFFTOOL_MODE_DEFINEFORSTACKS,   defineforstacksMode);
         MODECASE(XCFFTOOL_MODE_UPDATERUN,         updaterunMode);
         MODECASE(XCFFTOOL_MODE_TODO,              todoMode);
@@ -110 +108 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // I need the name of the current working database
+    psMetadataItem *dbnameMD = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING);
+    char *dbname = dbnameMD->data.str;
+
+    // required:
     PXOPT_LOOKUP_STR(workdir,     config->args, "-set_workdir", true, false);
     PXOPT_LOOKUP_STR(label,       config->args, "-set_label", true, false);
 
+    // we have two relevant databases: the current working database (e.g., gpc1) and the cross-camera
+    // database used to generate the cross-camera stacks (e.g., xcamera).  the cross-camera database
+    // name is supplied with -xcamera as an argument.  the current working database is part of the
+    // database config info
+
     // The xcamera must be supplied as a string, then the camera_id is selected
-    // from the table xcstackCamera.
+    // from the table xcstackCamera.  
     PXOPT_LOOKUP_STR(xcamera, config->args, "-xcamera", true, false);
 
-    // get camera_id for the working database:
-    // XXX need to extract the camera name (-dbname) from the config
-    psS64 camera_id = getCameraIDfromName(config, config->dbh->camera);
+    // get camera_id for the working database from the xcamera database
+    psS64 camera_id = getCameraIDfromName(config, xcamera, dbname);
     if (!camera_id) {
-      psError(PXTOOLS_ERR_SYS, false, "failed to find xcamera in database");
-      return false;
+        psError(PXTOOLS_ERR_SYS, false, "failed to find camera %s in database", dbname);
+        return false;
+    }
+    // get xcamera_id for the cross-camera database from the working database
+    psS64 xcamera_id = getCameraIDfromName(config, dbname, xcamera);
+    if (!xcamera_id) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to find camera %s in database", xcamera);
+        return false;
     }
 
@@ -131 +145 @@
     PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
 
-    // XXX is needed or wanted?
-    // PXOPT_LOOKUP_STR(sources_path_base,   config->args, "-set_sources_path_base", false, false);
-
     // first, we need to select the xccalRun(s) which need to have xcForceRuns.  we can limit the selection based on
     // qualities of the xccalRun or the xcstackRun.  There will be one xcForceRun for each
-    // identified xccalRun.
+    // identified xccalRun (there may be multiple xcForceRuns with different labels)
 
     psMetadata *xccalWhereMD = psMetadataAlloc();
@@ -149 +160 @@
     }
 
-    PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id",    "xcstackRun.skycell_id",      "LIKE");
-    PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id",       "xcstackRun.tess_id",         "==");
-    pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter",        "xcstackRun.filter",          "LIKE");
+    // the select query does not join to xcstackRun, but it does join to stackRun
+    PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id",    "stackRun.skycell_id",      "LIKE");
+    PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id",       "stackRun.tess_id",         "==");
+    pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter",        "stackRun.filter",          "LIKE");
     if (!pxskycellAddWhere(config, xccalWhereMD)) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -167 +179 @@
     pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group", "warpRun.data_group",      "LIKE");
     PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",         "warpRun.warp_id",         "==");
+    PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min",   "warpSkyfile.good_frac",  ">=");
     // XXX is is really necessary to limit the warps?
     if (!psListLength(warpWhereMD->list)) {
@@ -174 +187 @@
         return false;
     }
-    PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min",   "warpSkyfile.good_frac",  ">=");
 
     PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
@@ -180 +192 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
-    psString select = pxDataGet("xcfftool_definebyquery.sql");
-    if (!select) {
+    psString select = NULL;
+
+    psString query_pt1 = pxDataGet("xcfftool_definebyquery_pt1.sql");
+    if (!query_pt1) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(xccalWhereMD);
@@ -188 +202 @@
     }
 
+    // query_pt1 has 4 places where the xcamera database name must be supplied
+    psStringAppend (&select, query_pt1, xcamera, xcamera, xcamera, xcamera);
+    psFree (query_pt1);
+
+    if (!rerun) {
+        psStringAppend(&select, "LEFT JOIN xcForceRun ON (xcForceRun.xccal_id = %s.xccalRun.xccal_id) AND (xcForceRun.label = '%s')\n", xcamera, label);
+    }
+
+    psString query_pt2 = pxDataGet("xcfftool_definebyquery_pt2.sql");
+    if (!query_pt2) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(xccalWhereMD);
+        psFree(warpWhereMD);
+        psFree(select);
+        return false;
+    }
+
+    // query_pt2 has 3 places where the xcamera database name must be supplied and 1 for this camera
+    psStringAppend (&select, query_pt2, xcamera, (long long) camera_id, xcamera, xcamera);
+    psFree (query_pt2);
+
     psString where = NULL;
     psString whereClause = psDBGenerateWhereConditionSQL(xccalWhereMD, NULL);
-    psStringAppend(&where, "\nAND %s", whereClause);
-    psStringAppend(&select, " %s", where);
+    psStringAppend(&select, "\n AND %s", whereClause);
 
     psFree(whereClause);
     psFree(xccalWhereMD);
 
-    psString joinHook = NULL;
-    if (!rerun) {
-      psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id", xcamera);
-    }
-
-    psStringAppend(&joinHook, "\n %s, where);
-
-    if (
-        psStringAppend(&joinHook, "\n %s\nAND xcForceRun.label = '%s'", where, label);
-        psStringAppend(&select, "\nAND ff_id IS NULL");
-    }
-
-    // XXX need to add the xcamera name here to match definebyquery.sql
-    if (!p_psDBRunQueryF(config->dbh, select, joinHook)) {
+    // accept only the entries without a match in the target label
+    if (!rerun) psStringAppend(&select, "\n AND xcff_id IS NULL\n");
+
+    if (!p_psDBRunQuery(config->dbh, select)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(select);
@@ -215 +239 @@
     }
     psFree(select);
-    psFree(joinHook);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -251 +274 @@
     }
 
+    // this query selects all of the warps where were inputs to the given stackRun,
+    // with restrictions based on (quality, fault, warpRun.state, and other supplied conditions)
+    // there is no limit to the version of the warp analyss, to
     psString warpQueryTemplate = pxDataGet("xcfftool_definebyquery_select_warps.sql");
 
     whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL);
-    psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause);
+    psStringAppend(&warpQueryTemplate, "\n AND %s", whereClause);
 
     for (long i = 0; i < output->n; i++) {
@@ -268 +294 @@
         // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id");
         psS64 xccal_id = psMetadataLookupS64(&status, row, "xccal_id");
-
-        psString path_base = NULL;
-        if (sources_path_base) {
-            path_base = sources_path_base;
-        } else {
-            path_base = psMetadataLookupStr(&status, row, "path_base");
-            psAssert(status, "failed to find xccal path_base?");
-        }
-
+        psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id");
+        psString path_base = psMetadataLookupStr(&status, row, "path_base");
         psString xccal_data_group = psMetadataLookupStr(&status, row, "data_group");
         psString tess_id = psMetadataLookupStr(&status, row, "tess_id");
         psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id");
-        psString filter = psMetadataLookupStr(&status, row, "filter");
 
         psString query = NULL;
-        psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter);
+        psStringAppend(&query, warpQueryTemplate, (long long) stack_id);
 
         if (!p_psDBRunQuery(config->dbh, query)) {
@@ -315 +333 @@
         // create an xcForceRun
         if (!xcForceRunInsert(config->dbh,
-                                0x0,         // ff_id
-                                xccal_id,
-                                path_base,
-                                "new",       // state
-                                workdir,
-                                label,
-                                data_group ? data_group : (xccal_data_group ? xccal_data_group : label),
-                                dist_group,
-                                note,
-                                reduction,
-                                NULL        // registered
+                              0x0,           // xcff_id
+                              xccal_id,
+                              path_base,
+                              "new",         // state
+                              workdir,
+                              label,
+                              skycell_id,
+                              tess_id,
+                              data_group ? data_group : (xccal_data_group ? xccal_data_group : label),
+                              dist_group,
+                              note,
+                              reduction,
+                              NULL        // registered
                 )
             ) {
@@ -342 +362 @@
             psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id");
 
-            if (!xcForceInputInsert(config->dbh, ff_id, warp_id, xcamera_id)
-               ) {
+            if (!xcForceInputInsert(config->dbh, xcff_id, warp_id, xcamera_id)) {
                 if (!psDBRollback(config->dbh)) {
                     psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction");
@@ -372 +391 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-xcff_id",     "xcff_id",   "==");
-    PXOPT_COPY_STR(config->args, where, "-label",       "xcForceRun.label",    "==");
-    PXOPT_COPY_STR(config->args, where, "-data_group",  "xcForceRun.data_group",    "==");
-    PXOPT_COPY_STR(config->args, where, "-state",       "xcForceRun.state",    "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "stackRun.skycell_id",    "==");
+    PXOPT_COPY_S64(config->args, where, "-xcff_id",     "xcff_id",    "==");
+    PXOPT_COPY_STR(config->args, where, "-label",       "label",      "==");
+    PXOPT_COPY_STR(config->args, where, "-state",       "state",      "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group",  "data_group", "==");
+    PXOPT_COPY_STR(config->args, where, "-dist_group",  "dist_group", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",     "tess_id",    "==");
     if (!pxskycellAddWhere(config, where)) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -387 +408 @@
     }
 
-    psString query = pxDataGet("xcfftool_updaterun.sql");
-    if (!select) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
+    psString query = psStringCopy ("UPDATE xcForceRun JOIN skycell USING(tess_id, skycell_id)");
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "xcForceRun", "ff_id", "xcForceResult", true, false);
+    bool result = pxUpdateRun(config, where, &query, "xcForceRun", "xcff_id", "xcForceResult", true, false);
     psFree(query);
     psFree(where);
@@ -407 +424 @@
 
     psMetadata *whereMD = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, whereMD,  "-ff_id", "ff_id", "==");
-    pxAddLabelSearchArgs (config, whereMD, "-label", "xcForceRun.label", "==");
+    PXOPT_COPY_S64(config->args, whereMD,  "-xcff_id", "xcff_id", "==");
+    pxAddLabelSearchArgs (config, whereMD, "-label",   "xcForceRun.label", "==");
     pxskycellAddWhere(config, whereMD);
 
@@ -482 +499 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
     // required
-    PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false);
+    PXOPT_LOOKUP_S64(xcff_id, config->args, "-xcff_id", true, false);
     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
 
@@ -497 +514 @@
     // XXX not sure we need a transaction here...
     if (!xcForceResultInsert(config->dbh,
-                               ff_id,
+                               xcff_id,
                                warp_id,
                                path_base,
@@ -521 +538 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id",      "xcForceRun.ff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-xcff_id",    "xcForceRun.xcff_id", "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id",    "xcForceResult.warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
@@ -608 +625 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.ff_id", "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id", "xcForceResult.warp_id", "==");
     pxAddLabelSearchArgs(config, where, "-label", "xcForceRun.label", "==");
@@ -660 +677 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id",   "ff_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id",   "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id",   "==");
 
@@ -672 +689 @@
 }
 
-static bool setxcForceRunState(pxConfig *config, psS64 ff_id, const char *state)
-{
-    psString query = "UPDATE xcForceRun SET state = 'full' WHERE ff_id = %" PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, ff_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
 static bool toadvanceMode(pxConfig *config) {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
     psMetadata *whereMD = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, whereMD,  "-ff_id", "ff_id", "==");
+    PXOPT_COPY_S64(config->args, whereMD,  "-xcff_id", "xcff_id", "==");
     pxAddLabelSearchArgs (config, whereMD, "-label", "xcForceRun.label", "==");
     pxskycellAddWhere(config, whereMD);
@@ -765 +771 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
     // required
-    PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false);
+    PXOPT_LOOKUP_S64(xcff_id, config->args, "-xcff_id", true, false);
 
     // optional
@@ -783 +789 @@
 
     if (!xcForceSummaryInsert(config->dbh,
-                               ff_id,
+                               xcff_id,
                                path_base,
                                dtime_script,
@@ -799 +805 @@
 
     if (!fault) {
-        if (!setxcForceRunState(config, ff_id, "full")) {
+        if (!setxcForceRunState(config, xcff_id, "full")) {
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
@@ -817 +823 @@
 }
 
-static bool revertsummaryMode(pxConfig *config) {
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "==");
-    pxAddLabelSearchArgs(config, where, "-label", "xcForceRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-fault", "xcForceSummary.fault", "==");
-
-    if (!pxskycellAddWhere(config, where)) {
-        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
-        return false;
-    }
-
-    if (!psListLength(where->list)) {
-        psFree(where);
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        return false;
-    }
-
-    // Delete product
-    psString delete = pxDataGet("xcfftool_revertsummary.sql");
-    if (!delete) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&delete, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, delete)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(delete);
-        psFree(where);
-        return false;
-    }
-    psFree(delete);
-
-    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
-    psLogMsg("xcfftool", PS_LOG_INFO, "Deleted %d rows", numRows);
-
-    psFree(where);
-    return true;
-}
-static bool updatesummaryMode(pxConfig *config) {
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
-    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id",   "ff_id",   "==");
-
-
-    if (!pxSetFaultCode(config->dbh, "xcForceSummary", where, fault, quality)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-        psFree (where);
-        return false;
-    }
-    psFree (where);
-    return true;
-}
 static bool summaryMode(pxConfig *config) {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-ff_id",      "xcForceRun.ff_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "xcForceInput.warp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
-    PXOPT_COPY_S64(config->args, where, "-xccal_id",  "xcForceRun.xccal_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-stack_id",   "stackRun.stack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-xcff_id",    "xcForceRun.xcff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-xccal_id",   "xcForceRun.xccal_id", "==");
     PXOPT_COPY_STR(config->args, where, "-label",      "xcForceRun.label", "==");
     PXOPT_COPY_STR(config->args, where, "-data_group", "xcForceRun.data_group", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-filter",     "stackRun.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",    "xcForceRun.tess_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcForceRun.skycell_id", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault",      "xcForceSummary.fault", "==");
     pxskycellAddWhere(config, where);
@@ -966 +903 @@
 }
 
+static bool revertsummaryMode(pxConfig *config) {
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.xcff_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",   "xcForceRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",   "xcForceSummary.fault", "==");
+
+    if (!pxskycellAddWhere(config, where)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    // Delete product
+    psString delete = pxDataGet("xcfftool_revertsummary.sql");
+    if (!delete) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&delete, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, delete)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(delete);
+        psFree(where);
+        return false;
+    }
+    psFree(delete);
+
+    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
+    psLogMsg("xcfftool", PS_LOG_INFO, "Deleted %d rows", numRows);
+
+    psFree(where);
+    return true;
+}
+
+static bool updatesummaryMode(pxConfig *config) {
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-xcff_id",   "xcff_id",   "==");
+
+    if (!pxSetFaultCode(config->dbh, "xcForceSummary", where, fault, quality)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        psFree (where);
+        return false;
+    }
+    psFree (where);
+    return true;
+}
+
 bool exportrunMode(pxConfig *config)
 {
-  typedef struct ExportTable {
-    char tableName[80];
-    char sqlFilename[80];
-  } ExportTable;
+    typedef struct ExportTable {
+        char tableName[80];
+        char tableSQL[80];
+    } ExportTable;
 
     PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-    // PXOPT_LOOKUP_S64(det_id,  config->args, "-warp_id", true,  false);
     PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
     PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
-//    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
 
     FILE *f = fopen (outfile, "w");
@@ -994 +994 @@
 
     ExportTable tables [] = {
-      {"xcForceRun", "xcfftool_export_run.sql"},
-      {"xcForceInput", "xcfftool_export_input.sql"},
-      {"xcForceResult", "xcfftool_export_result.sql"},
-      {"xcForceSummary", "xcfftool_export_summary.sql"},
+        {"xcForceRun",     "SELECT xcForceRun.*       FROM xcForceRun"},
+        {"xcForceInput",   "SELECT xcForceInput.*     FROM xcForceInput"},
+        {"xcForceResult",  "SELECT xcForceResult.*    FROM xcForceResult"},
+        {"xcForceSummary", "SELECT xcForceSummary.*   FROM xcForceSummary"},
     };
 
@@ -1003 +1003 @@
 
     for (int i=0; i < numTables; i++) {
-      psString query = pxDataGet(tables[i].sqlFilename);
-      if (!query) {
-          psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-          return false;
-      }
-
-      if (where && psListLength(where->list)) {
-          psString whereClause = psDBGenerateWhereSQL(where, NULL);
-          psStringAppend(&query, " %s", whereClause);
-          psFree(whereClause);
-      }
-
-      // treat limit == 0 as "no limit"
-      if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-      }
-
-      if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-      }
-      psFree(query);
-
-      psArray *output = p_psDBFetchResult(config->dbh);
-      if (!output) {
-          psError(PS_ERR_UNKNOWN, false, "database error");
-          return false;
-      }
-      if (!psArrayLength(output)) {
-        psError(PS_ERR_UNKNOWN, true, "no rows found");
-        psFree(output);
-        return false;
-      }
-
-      // we must write the export table in non-simple (true) format
-      if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-      }
-      psFree(output);
+        psString query = psStringCopy(tables[i].tableSQL);
+
+        if (where && psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereSQL(where, NULL);
+            psStringAppend(&query, " %s", whereClause);
+            psFree(whereClause);
+        }
+
+        // treat limit == 0 as "no limit"
+        if (limit) {
+            psString limitString = psDBGenerateLimitSQL(limit);
+            psStringAppend(&query, " %s", limitString);
+            psFree(limitString);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+
+        psArray *output = p_psDBFetchResult(config->dbh);
+        if (!output) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        if (!psArrayLength(output)) {
+            psError(PS_ERR_UNKNOWN, true, "no rows found");
+            psFree(output);
+            return false;
+        }
+
+        // we must write the export table in non-simple (true) format
+        if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
     }
 
@@ -1054 +1050 @@
 }
 
+// XXX this is coded in a very strange way
 bool importrunMode(pxConfig *config)
 {
@@ -1067 +1064 @@
 
   psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
-
-#ifdef notdef
-  fprintf (stderr, "---- input ----\n");
-  psMetadataPrint (stderr, input, 1);
-#endif
 
   if (!pxCheckImportVersion(config, input)) {
@@ -1143 +1135 @@
 // We expect the camera to be defined, so an invalid name or a missing camera
 // results in a 0 return value which should be treated as an error.
-static psS64 getCameraIDfromName(pxConfig *config, const char *camera)
+static psS64 getCameraIDfromName(pxConfig *config, const char *dbname, const char *camera)
 {
     PS_ASSERT_PTR_NON_NULL(camera, false);
 
-    char *query = "SELECT camera_id FROM xcstackCamera WHERE camera = '%s'";
-    if (!p_psDBRunQueryF(config->dbh, query, camera)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to select camera_id for camera %s", camera);
+    char *query = "SELECT camera_id FROM %s.xcstackCamera WHERE camera = '%s'";
+    if (!p_psDBRunQueryF(config->dbh, query, dbname, camera)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to select camera_id for camera %s from database %s", camera, dbname);
         return 0;
     }
@@ -1197 +1189 @@
 }
 
-
+static bool setxcForceRunState(pxConfig *config, psS64 xcff_id, const char *state)
+{
+    psString query = "UPDATE xcForceRun SET state = 'full' WHERE xcff_id = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, xcff_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+

branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.h

@@ -26 +26 @@
     XCFFTOOL_MODE_NONE           = 0x0,
     XCFFTOOL_MODE_DEFINEBYQUERY,
-    XCFFTOOL_MODE_DEFINEFORSTACKS,
     XCFFTOOL_MODE_UPDATERUN,
     XCFFTOOL_MODE_TODO,

branches/eam_branches/ipp-20230313/ippTools/src/xcfftoolConfig.c

@@ -48 +48 @@
     // -definebyquery
     psMetadata *definebyqueryArgs = psMetadataAlloc();
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "search by warp label (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "search by warp data_group (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "search by warp ID", 0);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0);
-
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-xcamera", 0, "specify the cross-camera database", false);
+
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_xccal_label", PS_META_DUPLICATE_OK, "search by xccal label (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_xccal_data_group", PS_META_DUPLICATE_OK, "search by xccal data_group (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddS64(definebyqueryArgs,  PS_LIST_TAIL, "-select_xccal_id", 0, "search by xccal ID", 0);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "limits warps by label (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "limit warps by data_group (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddS64(definebyqueryArgs,  PS_LIST_TAIL, "-select_warp_id", 0, "limit warps by warp ID", 0);
     psMetadataAddF32(definebyqueryArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0);
+
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "limit by skycell_id (LIKE comparision)", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "limit by tess_id", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "limit by stack filter (LIKE comparison, multiple OK)", NULL);
 
     pxskycellAddArguments(definebyqueryArgs);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL);
@@ -72 +73 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
-    // -defineforstacks
-    psMetadata *defineforstacksArgs = psMetadataAlloc();
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddS64(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0);
-
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddF32(defineforstacksArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0);
-
-    pxskycellAddArguments(defineforstacksArgs);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_note", 0, "define note", NULL);
-    psMetadataAddStr(defineforstacksArgs,  PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL);
-    psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-rerun",  0, "queue new run even if one already exists for inputs", false);
-    psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
-    psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
-
     // -updaterun
     psMetadata *updaterunArgs = psMetadataAlloc();
-    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
-    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
-    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-xcff_id",    0, "search by xcff ID", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",      0, "search by label", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state",      0, "search by state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-tess_id",    0, "search by tess_id", NULL);
     pxskycellAddArguments(updaterunArgs);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
@@ -111 +91 @@
     // -todo
     psMetadata *todoArgs = psMetadataAlloc();
-    psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
+    psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0);
     psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     pxskycellAddArguments(todoArgs);
@@ -119 +99 @@
     // -addresult
     psMetadata *addresultArgs = psMetadataAlloc();
-    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0);
+    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0);
     psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0);
@@ -130 +110 @@
     // -result
     psMetadata *resultArgs= psMetadataAlloc();
-    psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
+    psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcforce ID", 0);
+    psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0);
     psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
     psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", 0);
     psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
-    psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0);
-    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0);
@@ -149 +128 @@
     // -revert
     psMetadata *revertArgs= psMetadataAlloc();
-    psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
+    psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0);
     psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
     psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
@@ -157 +136 @@
     // -updateresult
     psMetadata *updateresultArgs = psMetadataAlloc();
-    psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0);
+    psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0);
     psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0);
     psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0);
@@ -164 +143 @@
     // -toadvance
     psMetadata *toadvanceArgs = psMetadataAlloc();
-    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
+    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0);
     psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     pxskycellAddArguments(toadvanceArgs);
@@ -172 +151 @@
     // -addsummary
     psMetadata *addsummaryArgs = psMetadataAlloc();
-    psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0);
+    psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0);
     psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0);
     psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);
@@ -180 +159 @@
     psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
+    // -summary
+    psMetadata *summaryArgs= psMetadataAlloc();
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0);
+    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by skycal ID", 0);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0);
+    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0);
+    pxskycellAddArguments(summaryArgs);
+    psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -revertsummary
+    psMetadata *revertsummaryArgs= psMetadataAlloc();
+    psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0);
+    psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
+    psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    pxskycellAddArguments(revertsummaryArgs);
+
     // -updatesummary
     psMetadata *updatesummaryArgs = psMetadataAlloc();
-    psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0);
+    psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0);
     psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0);
     psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);
@@ -190 +189 @@
     psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
-    // -summary
-    psMetadata *summaryArgs= psMetadataAlloc();
-    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
-    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
-    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", 0);
-    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
-    psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
-    psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
-    pxskycellAddArguments(summaryArgs);
-    psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
-    psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
-    psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
-
-    // -revertsummary
-    psMetadata *revertsummaryArgs= psMetadataAlloc();
-    psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0);
-    psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
-    psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
-    pxskycellAddArguments(revertsummaryArgs);
-
     // -exportrun
     psMetadata *exportrunArgs = psMetadataAlloc();
-    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0,          "export this full force ID (required)", 0);
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-xcff_id", 0,          "export this xcff ID (required)", 0);
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
@@ -226 +200 @@
     psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
 
-
-
     psFree(now);
 
@@ -233 +205 @@
     psMetadata *modes   = psMetadataAlloc();
 
-    PXOPT_ADD_MODE("-definebyquery",   "Define a new full force run",                       XCFFTOOL_MODE_DEFINEBYQUERY,   definebyqueryArgs);
-    PXOPT_ADD_MODE("-defineforstacks", "Define a new full force runs for inputs to stacks", XCFFTOOL_MODE_DEFINEFORSTACKS, defineforstacksArgs);
+    PXOPT_ADD_MODE("-definebyquery",   "Define a new xcff run",                             XCFFTOOL_MODE_DEFINEBYQUERY,   definebyqueryArgs);
     PXOPT_ADD_MODE("-updaterun",       "Update a run",                                      XCFFTOOL_MODE_UPDATERUN,       updaterunArgs);
     PXOPT_ADD_MODE("-todo",            "Get runs to do",                                    XCFFTOOL_MODE_TODO,            todoArgs);

branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c

@@ -137 +137 @@
     psAssert (filters->type == PS_DATA_METADATA_MULTI, "-select_filter should be a multi container");
     psAssert (filters->data.list->n, "-select_filter should at least have a place-holder");
+
+# if (0)
+    // XXX 'filters' is a psMetadataItem of type MULTI.  Here is code to iterate over the list of items:
+    psListIterator *myIter = psListIteratorAlloc(filters->data.list, PS_LIST_HEAD, false);
+    psMetadataItem *myItem = NULL;
+    while ((myItem = psListGetAndIncrement (myIter))) {
+      fprintf (stderr, "type: %d\n", (int) myItem->type);
+    }
+# endif
+
     int num_filter = filters->data.list->n;
+
+    // if -select_filter was not supplied, then the 'filters' container will have a single entry
+    // with a value of NULL
+    if (num_filter == 1) {
+      // psListIterator *myIter = psListIteratorAlloc(filters->data.list, PS_LIST_HEAD, false);
+      // psMetadataItem *myItem = psListGetAndIncrement (myIter);
+
+      psMetadataItem *myItem = psListGet (filters->data.list, 0);
+      psAssert (myItem->type == PS_DATA_STRING, "unexpected data type for -select_filter");
+      if (myItem->data.str == NULL) {
+        psError(PXTOOLS_ERR_CONFIG, false, "at least one -select_filter must be supplied");
+        return false;
+      }
+    }
 
     PXOPT_LOOKUP_BOOL(group_by_data_group, config->args, "-group_by_data_group", false);