Changeset 42837 for trunk/ippTools/src

Timestamp:

May 9, 2025, 9:40:41 AM (14 months ago)

Author:

eugene

Message:

add support for xcstack, xcsky, xcff; update create (pxadmin) to be consistent with gpc1/gpc2; remove pstamp tables from pxadmin (not in gpc1/gpc2 db); fix a minor bug in addtool w.r.t. minidvodb_path; add -simple option to disttool; add updatesummitExp mode

Location:

trunk/ippTools

Files:

• . (modified) (1 prop)
• src (modified) (2 props)
• src/Makefile.am (modified) (3 diffs)
• src/addtool.c (modified) (3 diffs)
• src/disttool.c (modified) (5 diffs)
• src/disttoolConfig.c (modified) (2 diffs)
• src/fftool.c (modified) (4 diffs)
• src/pztool.c (modified) (6 diffs)
• src/pztoolConfig.c (modified) (1 diff)
• src/stacktool.c (modified) (1 diff)
• src/staticskytool.c (modified) (1 diff)
• src/xcfftool.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c )
• src/xcfftool.h (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.h )
• src/xcfftoolConfig.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcfftoolConfig.c )
• src/xcskytool.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c )
• src/xcskytool.h (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.h )
• src/xcskytoolConfig.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcskytoolConfig.c )
• src/xcstacktool.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcstacktool.c )
• src/xcstacktool.h (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcstacktool.h )
• src/xcstacktoolConfig.c (copied) (copied from branches/eam_branches/ipp-20230313/ippTools/src/xcstacktoolConfig.c )

trunk/ippTools/src

@@ -50 +50 @@
 remotetool
 fpcamtool
+xcstacktool
+xcskytool
+xcfftool

@@ -50 +50 @@
 remotetool
 fpcamtool
+xcstacktool
+xcskytool
+xcfftool

trunk/ippTools/src/Makefile.am

@@ -24 +24 @@
         stacktool \
         staticskytool \
+        xcstacktool \
+        xcskytool \
+        xcfftool \
         warptool \
         receivetool \
@@ -82 +85 @@
         regtool.h \
         stacktool.h \
+        xcstacktool.h \
+        xcskytool.h \
+        xcfftool.h \
         staticskytool.h \
         warptool.h \
@@ -245 +251 @@
     staticskytoolConfig.c
 
+xcstacktool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+xcstacktool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+xcstacktool_SOURCES = \
+    xcstacktool.c \
+    xcstacktoolConfig.c
+
+xcskytool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+xcskytool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+xcskytool_SOURCES = \
+    xcskytool.c \
+    xcskytoolConfig.c
+
+xcfftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+xcfftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+xcfftool_SOURCES = \
+    xcfftool.c \
+    xcfftoolConfig.c
+
 pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la

trunk/ippTools/src/addtool.c

@@ -1694 +1694 @@
   // required
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false);
-  PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false);
+  PXOPT_LOOKUP_STR(minidvodb_path_raw, config->args, "-set_minidvodb_path", false, false);
   PXOPT_LOOKUP_STR(minidvodb_host, config->args, "-set_minidvodb_host", false, false);
   //optional
-  PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false);
+  PXOPT_LOOKUP_STR(minidvodb_name_raw, config->args, "-set_minidvodb_name", false, false);
   PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
   PXOPT_LOOKUP_STR(note,            config->args, "-set_note",            false, false);
@@ -1706 +1706 @@
   }
 
-//  psString minidvodbpath = "NULL";
-
-  // I don't know how to get around the complaints of minidvodb_path can't be null. this 'fixes' it, but someone smarter can fix it properly.
-//  if (minidvodb_path) {
-//    minidvodbpath = minidvodb_path;
-//  } else {
-//    psError(PS_ERR_UNKNOWN, false, "require minidvodb_path");
-//    return false;
-//  }
-  
   if (!minidvodbRunInsert(config->dbh,
                           0, // job_id
-                          minidvodb_name,
+                          minidvodb_name_raw,
                           minidvodb_group,
-                          minidvodb_path,
+                          minidvodb_path_raw,
                           minidvodb_host,
                           "new",
@@ -1741 +1731 @@
   printf("%" PRId64 "\n", minidvodb_id);
 
-  
-  if (!minidvodb_name) {
-    psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id);
-  }
-  if (minidvodb_path) {
-    psStringAppend(&minidvodb_path,"/%s",minidvodb_name);
-  }
-
-  psString query = NULL;
-
-  psStringAppend(&query, "UPDATE minidvodbRun SET minidvodb_path = '%s', minidvodb_name = '%s' where minidvodb_id = %" PRIu64";", minidvodb_path, minidvodb_name, minidvodb_id);
-
-  if (!p_psDBRunQuery(config->dbh, query)) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query);
-    return false;
-  }
-
-  if (!psDBCommit(config->dbh)) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    return false;
-  }
+  // we created a new entry using the values supplied above for minidvodb_name_raw, minidvodb_host_raw.
+  // If minidvodb_name is not specified, it will be generated based on the value of minidvodb_group and
+  // the minidvodb_id of the new entry.  But this means we need to insert first, get the ID, then update.
+
+  bool doUpdate = false;
+
+  // Below: note that minidvodb_name_raw and minidvodb_path_raw are stored on config->args
+  // we need to work with a temporary external variable to avoid changing entries on the MD
+
+  // if minidvodb_name_raw is not specified above, create it from minidvodb_group
+  psString minidvodb_name = NULL;
+  if (minidvodb_name_raw) {
+    minidvodb_name = psStringCopy (minidvodb_name_raw);
+  } else {
+    psStringAppend(&minidvodb_name, "%s.%" PRIu64, minidvodb_group, minidvodb_id);
+    doUpdate = true;
+  }
+
+  // if minidvodb_path_raw IS specified above, update it with the minidvodb_name
+  psString minidvodb_path = NULL;
+  if (minidvodb_path_raw) {
+    psStringAppend(&minidvodb_path, "%s/%s", minidvodb_path_raw, minidvodb_name);
+    doUpdate = true;
+  }
+
+  if (doUpdate) {
+    psString query = NULL;
+
+    psStringAppend(&query, "UPDATE minidvodbRun SET minidvodb_path = '%s', minidvodb_name = '%s' where minidvodb_id = %" PRIu64";", minidvodb_path, minidvodb_name, minidvodb_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+  }
+
+  psFree (minidvodb_name);
+  psFree (minidvodb_path);
 
   return true;

trunk/ippTools/src/disttool.c

@@ -765 +765 @@
     }
 
-    if (simple) {/* no option? */}
-
     if (pretend) {
-        if (!ippdbPrintMetadatas(stdout, output, "distRunsToUpdate", true)) {
+        if (!ippdbPrintMetadatas(stdout, output, "distRunsToUpdate", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -1652 +1650 @@
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
     PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     distTargetRow *row = distTargetRowAlloc(
@@ -1676 +1675 @@
     row->target_id = psDBLastInsertID(config->dbh);
 
-    if (!distTargetPrintObject(stdout, row, true)) {
+    if (!distTargetPrintObject(stdout, row, !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print object");
             psFree(row);
@@ -1815 +1814 @@
     PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false);
     PXOPT_LOOKUP_STR(state, config->args,        "-set_state", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args,      "-simple", false);
 
     // XXX: should we insure that these names do not contatin any whitespace?
@@ -1842 +1842 @@
     row->dest_id = psDBLastInsertID(config->dbh);
 
-    if (!rcDestinationPrintObject(stdout, row, true)) {
+    if (!rcDestinationPrintObject(stdout, row, !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print object");
         psFree(row);

trunk/ippTools/src/disttoolConfig.c

@@ -275 +275 @@
     psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-last_fileset",0, "define last_fileset", NULL);
     psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-set_state",   0, "define state", NULL);
+    psMetadataAddBool(definedestinationArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
 
     // -updatedestination
@@ -292 +293 @@
     psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
     psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-comment",   0, "define comment", NULL);
+    psMetadataAddBool(definetargetArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
 
     // -updatetarget

trunk/ippTools/src/fftool.c

@@ -104 +104 @@
 }
 
-
+// NOTE: this mode seems to be insufficiently constrained.  It first identifies the
+// set of stackRun/skycalRun entries that lack a fullForceRun match.  But the
+// second stage is selecting all warps only based on (tess_id, skycell_id, filter) without
+// restriction to those which contributed to the stack
 static bool definebyqueryMode(pxConfig *config)
 {
@@ -150 +153 @@
     }
     PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min",  "warpSkyfile.good_frac",   ">=");
+    // XXX the tess_id and filter below can never be used because they are removed above (for stack)
+    // these options should be something like -select_warp_tess_id and -select_warp_filter
     PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",       "warpRun.tess_id",         "==");
     pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",        "rawExp.filter",          "LIKE");
@@ -223 +228 @@
     psString warpQueryTemplate = pxDataGet("fftool_definebyquery_select_warps.sql");
 
+    // NOTE: these allow the set of input warps to be restricted
     whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL);
     psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause);
@@ -339 +345 @@
 }
 
+// NOTE: this mode and 'definebyquery' seem to be nearly identical except that
+// 'definebyquery' allows for restrictions on the input warps by label, data_group,
+// warp_id, tess_id, filter, good_frac
 static bool defineforstacksMode(pxConfig *config)
 {

trunk/ippTools/src/pztool.c

@@ -50 +50 @@
 static bool updatenewexpMode(pxConfig *config);
 
-// XXX EAM : 2021.05.18 : code for updatesummitExp was added but not finished.
-// XXX static bool updatesummitExpMode(pxConfig *config);
+static bool updatesummitExpMode(pxConfig *config);
 
 // static bool copydoneCompleteExp(pxConfig *config);
@@ -93 +92 @@
         MODECASE(PZTOOL_MODE_UPDATEPZEXP, updatepzexpMode);
         MODECASE(PZTOOL_MODE_UPDATENEWEXP, updatenewexpMode);
-        // XXX MODECASE(PZTOOL_MODE_UPDATESUMMITEXP, updatesummitExpMode);
+        MODECASE(PZTOOL_MODE_UPDATESUMMITEXP, updatesummitExpMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -810 +809 @@
 }
 
-
 static bool updatecopiedMode(pxConfig *config)
 {
@@ -939 +937 @@
 }
 
-# if (0)
-static bool summitExpSetFault(pxConfig *config, const psS64 summit_id, const int fault)
+static bool summitExpSetFault(pxConfig *config, const psString exp_name, const psS64 summit_id, const int fault)
 {
     // check that state is a valid string value
@@ -949 +946 @@
   }
 
-    char *query = "UPDATE summitExp SET fault = %d WHERE summit_id = %ld";
-    if (!p_psDBRunQueryF(config->dbh, query, fault, summit_id)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to change state for %lld", (long long) summit_id);
+    char *query = "UPDATE summitExp SET fault = %d WHERE summit_id = %ld AND exp_name = '%s'";
+    if (!p_psDBRunQueryF(config->dbh, query, fault, summit_id, exp_name)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to change state for %s = %lld", exp_name, (long long) summit_id);
         return false;
     }
@@ -960 +957 @@
 static bool updatesummitExpMode(pxConfig *config)
 {
-  int fault;
-
+  // require both exp_name and summit_id as a double-check
   PS_ASSERT_PTR_NON_NULL(config, false);
   PXOPT_LOOKUP_S64(summit_id, config->args, "-summit_id", true, false);
-  PXOPT_LOOKUP_S32(state,    config->args, "-set_fault",true, false);
-
-  if (!summitExpSetFault(config,summit_id, fault)) {
-    psError(PS_ERR_UNKNOWN, false, "failed to change state for %lld", (long long) summit_id);
-    return false;
+  PXOPT_LOOKUP_STR(exp_name,  config->args, "-exp_name",  true, false);
+  PXOPT_LOOKUP_S32(fault,     config->args, "-set_fault", true, false);
+
+  if (!summitExpSetFault(config, exp_name, summit_id, fault)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to change state for %s = %lld", exp_name, (long long) summit_id);
+      return false;
   }
   return true;
 }
-# endif  
   
 static bool updatepzexpMode(pxConfig *config)

trunk/ippTools/src/pztoolConfig.c

@@ -171 +171 @@
     psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-set_state",  0,            "define new state (required)", NULL);
     
-# if (0)
     // -updatesummitexp
     psMetadata *updatesummitexpArgs = psMetadataAlloc();
     psMetadataAddS64(updatesummitexpArgs, PS_LIST_TAIL, "-summit_id", 0,     "define summit_id", 0);
     psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-exp_name",   0,            "search by exposure name (required)", NULL);
-    psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-inst",       0,            "search by camera (required)", NULL);
-    psMetadataAddStr(updatesummitexpArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope (required)", NULL);
     psMetadataAddS32(updatesummitexpArgs, PS_LIST_TAIL, "-set_fault",  0,            "define new fault (required)", 0);
-# endif
     
     // -updatenewexp

trunk/ippTools/src/stacktool.c

@@ -496 +496 @@
             psFree(insert);
             psFree(list);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
             return false;
         }

trunk/ippTools/src/staticskytool.c

@@ -1061 +1061 @@
     PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id",        "staticskyRun.sky_id",       "==");
     PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id",      "stackRun.stack_id",         "==");
-    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "==");
     PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id",       "stackRun.tess_id",          "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "LIKE");
     pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
     PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">=");

@@ -50 +50 @@
 remotetool
 fpcamtool
+xcstacktool
+xcskytool
+xcfftool