Changeset 33098 for branches/meh_branches/ppsub_test/ippTools/src

Timestamp:

Jan 11, 2012, 11:19:21 PM (15 years ago)

Author:

mhuber

Message:

merged latest r33093 trunk changes to meh_branches/ppsub_test

Location:

branches/meh_branches/ppsub_test

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (2 props)
• ippTools/src/Makefile.am (modified) (6 diffs)
• ippTools/src/addtool.c (modified) (35 diffs)
• ippTools/src/addtoolConfig.c (modified) (5 diffs)
• ippTools/src/bgtoolConfig.c (modified) (2 diffs)
• ippTools/src/chiptool.c (modified) (8 diffs)
• ippTools/src/chiptool.h (modified) (1 diff)
• ippTools/src/chiptoolConfig.c (modified) (2 diffs)
• ippTools/src/difftool.c (modified) (4 diffs)
• ippTools/src/difftoolConfig.c (modified) (1 diff)
• ippTools/src/disttool.c (modified) (13 diffs)
• ippTools/src/disttool.h (modified) (1 diff)
• ippTools/src/disttoolConfig.c (modified) (7 diffs)
• ippTools/src/faketool.c (modified) (2 diffs)
• ippTools/src/flatcorr.c (modified) (1 diff)
• ippTools/src/laptool.c (modified) (5 diffs)
• ippTools/src/laptool.h (modified) (1 diff)
• ippTools/src/laptoolConfig.c (modified) (2 diffs)
• ippTools/src/magicdstool.c (modified) (8 diffs)
• ippTools/src/magicdstoolConfig.c (modified) (2 diffs)
• ippTools/src/magictool.c (modified) (2 diffs, 1 prop)
• ippTools/src/magictoolConfig.c (modified) (1 diff)
• ippTools/src/mergetool.c (copied) (copied from trunk/ippTools/src/mergetool.c )
• ippTools/src/mergetool.h (copied) (copied from trunk/ippTools/src/mergetool.h )
• ippTools/src/mergetoolConfig.c (copied) (copied from trunk/ippTools/src/mergetoolConfig.c )
• ippTools/src/minidvodbtool.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (4 diffs)
• ippTools/src/pstamptool.h (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (2 diffs)
• ippTools/src/pxadd.c (modified) (3 diffs)
• ippTools/src/pxadd.h (modified) (1 diff)
• ippTools/src/pxmerge.c (copied) (copied from trunk/ippTools/src/pxmerge.c )
• ippTools/src/pxmerge.h (copied) (copied from trunk/ippTools/src/pxmerge.h )
• ippTools/src/pxtools.c (modified) (1 diff)
• ippTools/src/regtool.c (modified) (5 diffs)
• ippTools/src/regtoolConfig.c (modified) (1 diff)
• ippTools/src/staticskytool.c (modified) (6 diffs)
• ippTools/src/staticskytool.h (modified) (1 diff)
• ippTools/src/staticskytoolConfig.c (modified) (3 diffs)
• ippTools/src/vptool.c (copied) (copied from trunk/ippTools/src/vptool.c )
• ippTools/src/vptool.h (copied) (copied from trunk/ippTools/src/vptool.h )
• ippTools/src/vptoolConfig.c (copied) (copied from trunk/ippTools/src/vptoolConfig.c )
• ippTools/src/warptool.c (modified) (7 diffs)
• ippTools/src/warptoolConfig.c (modified) (2 diffs)

branches/meh_branches/ppsub_test/ippTools/src

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool

branches/meh_branches/ppsub_test/ippTools/src/Makefile.am

@@ -29 +29 @@
         diffphottool \
         minidvodbtool \
-        laptool
+        mergetool \
+        laptool \
+        vptool
 
 pkginclude_HEADERS = \
@@ -42 +44 @@
         pxmagic.h \
         pxminidvodb.h \
+        pxmerge.h \
         pxregister.h \
         pxspace.h \
@@ -78 +81 @@
         diffphottool.h \
         minidvodbtool.h \
-        laptool.h
+        mergetool.h \
+        laptool.h \
+        vptool.h
 
 lib_LTLIBRARIES = libpxtools.la
@@ -96 +101 @@
         pxmagic.c \
         pxminidvodb.c \
+        pxmerge.c \
         pxregister.c \
         pxspace.c \
@@ -292 +298 @@
     minidvodbtoolConfig.c
 
+mergetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+mergetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+mergetool_SOURCES = \
+    mergetool.c \
+    mergetoolConfig.c
+
 laptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 laptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
@@ -298 +310 @@
     laptoolConfig.c
 
+vptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+vptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+vptool_SOURCES = \
+    vptool.c \
+    vptoolConfig.c
+
 clean-local:
         -rm -f TAGS

branches/meh_branches/ppsub_test/ippTools/src/addtool.c

@@ -143 +143 @@
     pxAddLabelSearchArgs (config, where, "-filter",     "stackRun.filter", "=="); // define using camRun label
     }
-    
+        if (strcmp(stage, "staticsky_multi")== 0) {
+
+    pxAddLabelSearchArgs (config, where, "-label",     "staticskyRun.label", "=="); // define using camRun label
+    pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label
+    PXOPT_COPY_STR(config->args, where,  "-reduction", "staticskyyRun.reduction", "==");
+    //no filter here
+    }
     if (!psListLength(where->list)) {
         psFree(where);
@@ -237 +243 @@
     }
 
-
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      if (dvodb ) {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql");
+        // user supplied dvodb
+        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
+    } else {
+      psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi.sql");
+        // inherit dvodb from camRun, avoid matching NULL
+        psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)");
+    }
+    }
 
     if (!bare_query) {
@@ -272 +292 @@
     } else {
       //This picks only the unmagicked/uncensored ones
-      if (strcmp(stage,"cam") == 0) {
-        psStringAppend(&query, " AND (camRun.magicked = 0)");
-      }
+      //if (strcmp(stage,"cam") == 0) {
+        //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored.
+        //psStringAppend(&query, " AND (camRun.magicked = 0)");
+        //}
       if (strcmp(stage,"stack") == 0) {
         psStringAppend(&query, " AND (stackRun.magicked = 0)");
@@ -280 +301 @@
 
     }
+
+    //if we grab a group of camRun/stackRuns that have multiple camRun/stackRun pointing to the same exp_id/stack_id, and that exp_id/stack_id has never before been added to addRun, we need to group by exp_id/stack_id to ensure that only 1 of the camRun/stackRun for that exp_id is added to the addRun stage.
+
+    if (strcmp(stage,"cam") == 0) {
+        psStringAppend(&query, " GROUP BY exp_id"); 
+   }
+      if (strcmp(stage,"stack") == 0) {
+        psStringAppend(&query, " GROUP BY stack_id");
+      }
+    if (strcmp(stage,"staticsky") == 0) {
+        psStringAppend(&query, " GROUP BY stack_id");
+      }
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      psStringAppend(&query, " GROUP BY sky_id");  //some reason it needs this
+      }
+
 
     psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query);
@@ -391 +428 @@
     }
     }
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        if (num_inputs < 0) {
+          psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs");
+          return false;
+        }
+
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        if (!dvodb) {  //there's no staticsky.dvodb
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+        if (!workdir && !row->workdir) {
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+
+        psFree(row);
+    }
+    }
+
+
+
 
     // start a transaction so we don't end up with an exp without any associted
@@ -424 +500 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -465 +542 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -506 +584 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -530 +609 @@
       }
 
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psS64 stage_id =0; 
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        stage_id = row->sky_id;
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        for (int filter_id = 0; filter_id < num_inputs; filter_id++) {  
+        if (!pxaddQueueByCamID(config,
+                               stage,
+                               stage_id,
+                               filter_id,
+                               workdir     ? workdir   : row->workdir,
+                               reduction   ? reduction : row->reduction,
+                               label       ? label     : row->label,
+                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
+                               dvodb       ? dvodb     : NULL,
+                               note        ? note      : NULL,
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
+            psFree(row);
+            psFree(output);
+            return false;
+        }
+        }
+        psFree(row);
+    }
+      }
+
+
+
+
     psFree(output);
 
@@ -571 +703 @@
     }
     if (strcmp(stage, "staticsky")==0) {
+    query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
+    }
+    if (strcmp(stage, "staticsky_multi")==0) {
     query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
     }
@@ -612 +747 @@
     query = pxDataGet("addtool_find_pendingexp_staticsky.sql");
     }
-
+    if (strcmp(stage, "staticsky_multi")==0) { 
+    query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql");
+    }
+    
 
     if (!query) {
@@ -634 +772 @@
       psStringAppend(&query, " GROUP BY %s", "sky_id");
     }
-
+    if (strcmp(stage, "staticsky_multi") == 0) {
+      //this group by is needed to join against all the warps (to get camera)
+      psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1");
+    }
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -684 +825 @@
     PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false);
     PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
-
+    PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false);
     // generate restrictions
     psMetadata *where = psMetadataAlloc();
@@ -762 +903 @@
       }
     }
-
+    //if there is a stage_extra1, set it in addRun (it's not known until it is processed)
+    if (stage_extra1) {
+      psString setExtra = NULL;
+      psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1);
+      psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id);
+
+      if (!p_psDBRunQuery(config->dbh, setExtra)) {
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+
+        return false;
+      }
+    }
 
 
@@ -824 +979 @@
     query = pxDataGet("addtool_find_processedexp_staticsky.sql");
     }
+    if (strcmp (stage,"staticsky_multi") == 0) {
+    query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql");
+    }
+
    
     if (!query) {
@@ -905 +1064 @@
     PXOPT_COPY_S64(config->args, where, "-add_id",    "addRun.add_id",         "==");
     PXOPT_COPY_S64(config->args, where, "-stage_id",    "addRun.stage_id",         "==");
-    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
+    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", true, false);
     pxcamGetSearchArgs (config, where);
     pxAddLabelSearchArgs (config, where, "-label",    "addRun.label",     "==");
@@ -934 +1093 @@
          query = pxDataGet("addtool_revertprocessedexp_staticsky.sql");
       }
+   if (strcmp(stage, "staticsky_multi") == 0) {
+         query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql");
+      }
+
+
+     
       if (!query) {
             // rollback
@@ -1071 +1236 @@
 
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false);
-
-  //optional
+    //optional
   PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false);
   PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false);
@@ -1088 +1251 @@
   if (minidvodb_path) {
     minidvodbpath = minidvodb_path;
+  } else {
+    psError(PS_ERR_UNKNOWN, false, "require minidvodb_path");
+    return false;
   }
 
@@ -1095 +1261 @@
             minidvodb_group,
             minidvodbpath,
-            mergedvodb_path,
-            "new",
+            "new",
             0
             )) {
@@ -1118 +1283 @@
     psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id);
   }
-
-  if (!minidvodb_path) {
-    psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name);
-  }
-
   if (minidvodb_path) {
     psStringAppend(&minidvodb_path,"/%s",minidvodb_name);
@@ -1154 +1314 @@
     PXOPT_COPY_STR(config->args, where, "-state",     "state", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "==");
-    PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_group",     "minidvodb_group", "==");
 
@@ -1160 +1319 @@
     PXOPT_LOOKUP_STR(minidvodb_path,  config->args, "-set_minidvodb_path", false, false);
     PXOPT_LOOKUP_STR(state,  config->args, "-set_state", false, false);
-    PXOPT_LOOKUP_STR(mergedvodb_path,  config->args, "-set_mergedvodb_path", false, false);
     PXOPT_LOOKUP_STR(minidvodb_group,  config->args, "-set_minidvodb_group", false, false);
 
@@ -1195 +1353 @@
     }
 
-    if (mergedvodb_path) {
-      if (cnt) {
-        psStringAppend(&query, "%s", comma);
-      }
-      psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path);
-      cnt++;
-    }
-
     if (minidvodb_group) {
       if (cnt) {
@@ -1246 +1396 @@
   }
 
-  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL");
+  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL");
 
   psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL);
@@ -1294 +1444 @@
 
   //now flip new -> active
-  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL ");
+  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL ");
   psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group);
   if (!p_psDBRunQuery(config->dbh, query2)) {
@@ -1471 +1621 @@
     // required
   PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false);
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false);
 
     // optional
-  PXOPT_LOOKUP_U64(merge_order,     config->args, "-merge_order", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false);
-  PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false);
   PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
   
@@ -1530 +1676 @@
   minidvodbProcessedRow *row = minidvodbProcessedRowAlloc(
                pendingRow->minidvodb_id,
-               merge_order,
                dtime_resort,
                dtime_relphot,
-               dtime_verify,
-               dtime_merge,
                dtime_script,
                epoch,
-               mergedvodb_path,
                fault
                );
@@ -1552 +1694 @@
   }
 
-
-
-
-  //this finds the # of merged things (for the merge order)
-  psString query3 = NULL;
-  psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group);
-
-  if (!p_psDBRunQuery(config->dbh, query3)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query3);
-    return false;
-  }
-  psArray *output2 = p_psDBFetchResult(config->dbh);
-  if (!output2) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    return false;
-  }
-  if (!psArrayLength(output2)) {
-    psTrace("addtool", PS_LOG_INFO, "no rows found");
-    psFree(output);
-    return true;
-  }
-  bool status;
-  psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)");
-  if (!status) {
-
-    psAbort("failed to lookup value for count column");
-    return false;
-  }
-  psString final = NULL;
-  psStringAppend(&final, "%" PRIu64, m_order);
-    //return false;
-  psFree(query3);
-  psFree(output2);
-
-
-
-  //update the merge_order
-
-  psString query4 = NULL;
-  psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order);
-  psStringAppend(&query4," where minidvodb_id = %" PRIu64,  minidvodb_id);
-  //printf("%s", query4);
-  if (!p_psDBRunQuery(config->dbh, query4)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query4);
-    return false;
-  }
-  //
-psFree(query4);
-
-
-
-// since there is only one exp per 'new' set mindvodbRun.state = 'merged'
-
   psString query2 = NULL ;
   psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id);
@@ -1627 +1706 @@
     return false;
   }
-
-
-
-
-
-
-
-
   psFree(row);
   psFree(pendingRow);
@@ -1647 +1718 @@
     return false;
   }
-
-
-  //print the merge_order (why not!)
-  printf("%s", final);
-  psFree(final);
-
   return true;
 }
@@ -1661 +1726 @@
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "==");
-  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "==");
-  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "==");
   PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
   PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
@@ -1811 +1876 @@
 
   PXOPT_LOOKUP_U64(minidvodb_id,  config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_U64(merge_order,  config->args, "-set_merge_order", false, false);
   PXOPT_LOOKUP_S16(fault,  config->args, "-set_fault", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot,  config->args, "-set_dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort,  config->args, "-set_dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge,  config->args, "-set_dtime_merge", false, false);
-PXOPT_LOOKUP_F32(dtime_verify,  config->args, "-set_dtime_verify", false, false);
 PXOPT_LOOKUP_F32(dtime_script,  config->args, "-set_dtime_script", false, false);
   PXOPT_COPY_S64(config->args, where, "-minidvodb_id",     "minidvodbProcessed.minidvodb_id", "==");
@@ -1836 +1898 @@
   }
 
-  if (merge_order) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-
-    psStringAppend(&query, " merge_order = %" PRId64, merge_order);
-    cnt++;
-  }
-
   if (dtime_relphot) {
     if (cnt) {
@@ -1861 +1914 @@
   }
 
-  if (dtime_merge) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_merge = %f", dtime_merge);
-    cnt++;
- }
-if (dtime_verify) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_verify = %f", dtime_verify);
-    cnt++;
- }
-if (dtime_script) {
+  if (dtime_script) {
     if (cnt) {
       psStringAppend(&query, "%s", comma);

branches/meh_branches/ppsub_test/ippTools/src/addtoolConfig.c

@@ -115 +115 @@
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN);
     psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1",  0,            "set stage_extra1", 0);
+
 
     // -processedexp
@@ -165 +167 @@
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define minidvodb_group (required)", NULL);
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
-    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for the merged dvodb (required)", NULL);
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
 
@@ -174 +175 @@
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name (LIKE)", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path",        0, "search by path for minidvodb", NULL);
-    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path",        0, "search by path for the merged dvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state",        0, "search by state", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name",        0, "define minidvodb_name", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
-    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for minidvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define path for the merged dvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
@@ -205 +204 @@
     psMetadata *addminidvodbprocessedArgs = psMetadataAlloc();
     psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0,    "define minidvodb_id (required)", 0);
-    psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order",    0,    "define merge order", 0);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot",  0,    "define elapsed time for relphot (seconds)", NAN);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0,    "define elapsed time for resort (seconds)", NAN);
-    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify",    0,    "define elapsed time for DVO verify (seconds)", NAN);
-    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", NAN);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script",    0,    "define elapsed time for script (seconds)", NAN);
     psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch",         0,    "time merge is finished", NULL);
-    psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0,    "path of merged dvodb", NULL);
     psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0,    "minidvodb_group", NULL);
     psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault",          0,    "set fault code", 0);
@@ -232 +227 @@
     psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name", NULL);
     psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault",  0,            "set fault code", 0);
-    psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order",    0,    "define merge order", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot",  0,    "define elapsed time for relphot (seconds)", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0,    "define elapsed time for resort (seconds)", 0);
-    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify",    0,    "define elapsed time for DVO verify (seconds)", 0);
-    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script",    0,    "define elapsed time for script (seconds)", 0);
     psMetadata *argSets = psMetadataAlloc();

branches/meh_branches/ppsub_test/ippTools/src/bgtoolConfig.c

@@ -113 +113 @@
     psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
     psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0); 
+    psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-not_destreaked",  0,        "define magicked status", 0); 
     psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
     psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
@@ -298 +300 @@
     psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
     psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0); 
     psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
     psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);

branches/meh_branches/ppsub_test/ippTools/src/chiptool.c

@@ -43 +43 @@
 static bool revertprocessedimfileMode(pxConfig *config);
 static bool updateprocessedimfileMode(pxConfig *config);
+static bool dropprocessedimfileMode(pxConfig *config);
 static bool advanceexpMode(pxConfig *config);
 static bool blockMode(pxConfig *config);
@@ -88 +89 @@
         MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,   revertprocessedimfileMode);
         MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,   updateprocessedimfileMode);
+        MODECASE(CHIPTOOL_MODE_DROPPROCESSEDIMFILE,     dropprocessedimfileMode);
         MODECASE(CHIPTOOL_MODE_ADVANCEEXP,              advanceexpMode);
         MODECASE(CHIPTOOL_MODE_BLOCK,                   blockMode);
@@ -883 +885 @@
     }
     psFree(query);
+    int numDeleted = psDBAffectedRows(config->dbh);
+    psLogMsg("chiptool", PS_LOG_INFO, "Deleted %d chipProcessedImfiles", numDeleted);
 
     if (!fault) {
@@ -896 +900 @@
     }
     psFree(query_update);
+    int numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("chiptool", PS_LOG_INFO, "Updated %d chipProcessedImfiles", numUpdated);
+
 
     return true;
@@ -962 +969 @@
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
-    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
     if (psListLength(where->list) == 0) {
         psError(PS_ERR_UNKNOWN, true, "search parameters are required");
@@ -986 +993 @@
 
 
+    return true;
+}
+
+// "drop" previously processed components as though the original processing
+// had poor quality. Sets quality, data_state, clears fault and magicked
+// Deletes corresponding magicDSFile (if one exists)
+static bool dropprocessedimfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
+    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false);
+    PXOPT_LOOKUP_S64(class_id, config->args, "-class_id", true, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", true, false);
+
+    if (!quality) {
+        psError(PXTOOLS_ERR_CONFIG, true, "non-zero quality value is required");
+        return false;
+    }
+
+    if (psListLength(where->list) == 0) {
+        // won't get here
+        psError(PXTOOLS_ERR_CONFIG, true, "-chip_id and -class_id are required");
+        return false;
+    }
+
+    psString query = pxDataGet("chiptool_dropprocessedimfile_update.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+    psString delete_query = pxDataGet("chiptool_dropprocessedimfile_delete.sql");
+    if (!delete_query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(where);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!p_psDBRunQueryF(config->dbh, query, quality)) {
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error on rollback");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psStringAppend(&delete_query, " AND %s", whereClause);
+    psFree(whereClause);
+    if (!p_psDBRunQuery(config->dbh, delete_query)) {
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error on rollback");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(delete_query);
+        return false;
+    }
+    psFree(delete_query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
     return true;
 }
@@ -1418 +1497 @@
     psMetadata *where = psMetadataAlloc();
     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
-
+    PXOPT_COPY_S64(config->args, where,  "-chip_id",    "chip_id", "==");
     // look for completed chipPendingExp
     // migrate them to chipProccessedExp & camPendingExp
@@ -1432 +1511 @@
         psFree(whereClause);
     }
+
+    psTrace("chiptool", 10, "%s\n", query);
     psFree(where);
 

branches/meh_branches/ppsub_test/ippTools/src/chiptool.h

@@ -33 +33 @@
     CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,
     CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,
+    CHIPTOOL_MODE_DROPPROCESSEDIMFILE,
     CHIPTOOL_MODE_SETIMFILETOUPDATE,
     CHIPTOOL_MODE_ADVANCEEXP,

branches/meh_branches/ppsub_test/ippTools/src/chiptoolConfig.c

@@ -237 +237 @@
     psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0,         "set state", NULL);
     psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_quality",  0,            "set quality", 0);
-    // -processedExp
+
+    // -dropprocessedimfile
+    psMetadata *dropprocessedimfileArgs = psMetadataAlloc();
+    psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id",  0,            "search by chip ID (required)", 0);
+    psMetadataAddStr(dropprocessedimfileArgs,  PS_LIST_TAIL, "-class_id",           0, "search by class ID (required)", NULL);
+    psMetadataAddS16(dropprocessedimfileArgs, PS_LIST_TAIL, "-set_quality",  0,        "set quality (required)", 0);
+
+    // -listrun
     psMetadata *listrunArgs = psMetadataAlloc();
     pxchipSetSearchArgs(listrunArgs);
@@ -372 +379 @@
     PXOPT_ADD_MODE("-processedimfile",      "show processed imfiles",               CHIPTOOL_MODE_PROCESSEDIMFILE,      processedimfileArgs);
     PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties",    CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs);
+    PXOPT_ADD_MODE("-dropprocessedimfile","drop previously procesed imfile",        CHIPTOOL_MODE_DROPPROCESSEDIMFILE,  dropprocessedimfileArgs);
     PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile",              CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs);
     PXOPT_ADD_MODE("-runstate",             "list the states of chip run",          CHIPTOOL_MODE_RUNSTATE,             runstateArgs);

branches/meh_branches/ppsub_test/ippTools/src/difftool.c

@@ -1634 +1634 @@
     PXOPT_COPY_F32(config->args, selectWhere, "-timediff",
                    "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");
+    PXOPT_COPY_F32(config->args, selectWhere, "-mintimediff",
+                   "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", ">=");
 
     // other where restrictions:
@@ -1867 +1869 @@
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
         psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group");
-        if (input_data_group) {
+        if (!data_group) {
           data_group = input_data_group;
         }
@@ -1934 +1936 @@
             return false;
         }
+        psS64 numSkycells = psDBAffectedRows(config->dbh);
+        if (numSkycells > 0) {
+            if (!setdiffRunState(config, run->diff_id, "new", false)) {
+                psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64,
+                        run->diff_id);
+                psFree(input);
+                psFree(diff);
+                psFree(run);
+                psFree(list);
+                psFree(insert);
+                psFree(results);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+            run->state = psStringCopy("new");
+        } else {
+            // No overlap between the warps. Insert a dummy skycell which will complete this pair of warps
+            // XXX: we'd like to use diffInputSkyfileRowAlloc and diffInputSkyfileInsert but there doesn't
+            // seem to be a way to pass in NULL for skycell_id, stack1 and stack2
+            psString dummyQuery = NULL;
+            psStringAppend(&dummyQuery, 
+              "INSERT INTO diffInputSkyfile VALUES(%" PRId64 ", NULL, %s, NULL, %s, NULL, '%s', 0)",
+                    run->diff_id, 
+                    input,                      // warp1
+                    template,                   // warp2
+                    tess_id);
+            if (!p_psDBRunQuery(config->dbh, dummyQuery)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(dummyQuery);
+                psFree(input);
+                psFree(diff);
+                psFree(run);
+                psFree(list);
+                psFree(insert);
+                psFree(results);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+            psFree(dummyQuery);
+            psString finishQuery = NULL;
+            psStringAppend( &finishQuery, 
+                "UPDATE diffRun set state ='full', note = 'dummy run - no overlap' WHERE diff_id = %" PRId64,
+                run->diff_id);
+            if (!p_psDBRunQuery(config->dbh, finishQuery)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(finishQuery);
+                psFree(input);
+                psFree(diff);
+                psFree(run);
+                psFree(list);
+                psFree(insert);
+                psFree(results);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+            }
+            psFree(finishQuery);
+            run->state = psStringCopy("full");
+        }
         psFree(input);
         psFree(diff);
-
-        if (!setdiffRunState(config, run->diff_id, "new", false)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64,
-                    run->diff_id);
-            psFree(run);
-            psFree(list);
-            psFree(insert);
-            psFree(results);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
 
         psArrayAdd(list, list->n, run);
@@ -2461 +2514 @@
         psString limitString = psDBGenerateLimitSQL(limit);
         psStringAppend(&query, " %s", limitString);
+        psStringSubstitute(&query,limitString,"@INNERLIMITS@");
         psFree(limitString);
+
+    }
+    else {
+      psStringSubstitute(&query,NULL,"@INNERLIMITS@");
     }
     //    fprintf(stderr,"%s",query);

branches/meh_branches/ppsub_test/ippTools/src/difftoolConfig.c

@@ -283 +283 @@
     psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
     psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN);
+    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-mintimediff", 0, "limit time difference between input and template to be greater than this", NAN);
     psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-backwards", 0, "Template comes after input?", false);
     psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN);

branches/meh_branches/ppsub_test/ippTools/src/disttool.c

@@ -34 +34 @@
 static bool updaterunMode(pxConfig *config);
 static bool revertrunMode(pxConfig *config);
-static bool rerunMode(pxConfig *config);
+static bool startoverMode(pxConfig *config);
 static bool pendingcomponentMode(pxConfig *config);
 static bool addprocessedcomponentMode(pxConfig *config);
+static bool updateprocessedcomponentMode(pxConfig *config);
+static bool revertcomponentMode(pxConfig *config);
 static bool revertcomponentMode(pxConfig *config);
 static bool processedcomponentMode(pxConfig *config);
@@ -84 +86 @@
         MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode);
         MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode);
-        MODECASE(DISTTOOL_MODE_RERUN, rerunMode);
+        MODECASE(DISTTOOL_MODE_STARTOVER, startoverMode);
         MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode);
         MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode);
+        MODECASE(DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentMode);
         MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode);
         MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode);
@@ -141 +144 @@
     PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false);
     PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
-    PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", false);
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false);
@@ -165 +167 @@
     PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false);
 
-    // queuing no_magic runs can be quite dangerous. Require confirmation
-    if (no_magic && !pretend && !i_am_sure) {
-        psError(PXTOOLS_ERR_SYS, true, "-i_am_sure required to queue no_magic distribution runs");
-        return false;
-    }
-   
+    PXOPT_LOOKUP_BOOL(single, config->args, "-singlefilter", false);
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+
     if (use_alternate) {
         if (strcmp(stage, "raw")) {
@@ -361 +360 @@
     } else if (!strcmp(stage, "sky")) {
         magicRunType = "staticskyRun";
-        query = pxDataGet("disttool_definebyquery_sky.sql");
+        if (single) {
+            query = pxDataGet("disttool_definebyquery_sky_singlefilter.sql");
+        } else {
+            query = pxDataGet("disttool_definebyquery_sky.sql");
+        }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -406 +409 @@
         }
     }
+    if (!rerun) {
+        psStringAppend(&query, " AND (distRun.dist_id IS NULL)");
+    }
 
     if (psListLength(where->list)) {
@@ -418 +424 @@
     if (!no_magic) {
         psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType);
+    }
+
+    if (!strcmp(stage, "sky")) {
+        if (single) {
+            psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1");
+        } else {
+            psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) > 1");
+        }
     }
 
@@ -636 +650 @@
 }
 
-static bool rerunMode(pxConfig *config)
+static bool startoverMode(pxConfig *config)
 {
     psMetadata *where = psMetadataAlloc();
@@ -646 +660 @@
 
     // require data_group or dist_id to be supplied
-    PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false);
     PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false);
     if (!data_group && !dist_id) {
@@ -771 +785 @@
     PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "distRun.label", "==");
 
     PXOPT_COPY_S16(config->args, where,  "-fault", "distRun.fault", "==");
@@ -962 +976 @@
     return true;
 }
+static bool updateprocessedcomponentMode(pxConfig *config)
+{
+
+    // required values
+    PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false);
+    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
+
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
+    PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false);
+    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
+    PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", false, false);
+    PXOPT_LOOKUP_STR(name, config->args, "-name", false, false);
+
+    bool setfault = clearfault || fault;
+
+    if (!bytes && !md5sum && !outdir && !name && !setfault && !fault) {
+        psError(PS_ERR_UNKNOWN, true, "at least one of bytes md5sum outdir name fault or setfault is required");
+        return false;
+    }
+
+    char *sep = "";
+    psString query = psStringCopy("UPDATE distComponent SET ");
+    if (setfault) {
+        psStringAppend(&query, "%s fault = %d", sep, fault ? 1 : 0);
+        sep = ", ";
+    }
+    if (bytes) {
+        psStringAppend(&query, "%s bytes = %d", sep, bytes);
+        sep = ", ";
+    }
+    if (md5sum) {
+        psStringAppend(&query, "%s md5sum = '%s'", sep, md5sum);
+        sep = ", ";
+    }
+    if (outdir) {
+        psStringAppend(&query, "%s outdir = '%s'", sep, outdir);
+        sep = ", ";
+    }
+    if (name) {
+        psStringAppend(&query, "%s name = '%s'", sep, name);
+        sep = ", ";
+    }
+
+    psStringAppend(&query, "\nWHERE dist_id = %"PRId64 " AND component = '%s'",
+        dist_id, component);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
 
 static bool toadvanceMode(pxConfig *config)
@@ -1041 +1111 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -2178 +2249 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
 
     if (!psListLength(where->list)) {

branches/meh_branches/ppsub_test/ippTools/src/disttool.h

@@ -28 +28 @@
     DISTTOOL_MODE_UPDATERUN,
     DISTTOOL_MODE_REVERTRUN,
-    DISTTOOL_MODE_RERUN,
+    DISTTOOL_MODE_STARTOVER,
     DISTTOOL_MODE_PENDINGCOMPONENT,
     DISTTOOL_MODE_ADDPROCESSEDCOMPONENT,
     DISTTOOL_MODE_REVERTCOMPONENT,
     DISTTOOL_MODE_PROCESSEDCOMPONENT,
+    DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT,
     DISTTOOL_MODE_TOADVANCE,
     DISTTOOL_MODE_PENDINGCLEANUP,

branches/meh_branches/ppsub_test/ippTools/src/disttoolConfig.c

@@ -49 +49 @@
 
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirmation needed when -no_magic is supplied", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label for run", NULL);
@@ -64 +63 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",         0, "select by run label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group",    0, "select by dist_group", NULL);
-
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-singlefilter", 0, "select single filter runs (sky stage only)", false);
+
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new distRun even if one already exists", false);
     psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
@@ -87 +88 @@
     psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault",      0, "define fault code", 0);
 
-    // -rerun
-    psMetadata *rerunArgs = psMetadataAlloc();
-    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group",    0, "define data_group (required)", NULL);
-    psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0);
-    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage",    0, "search by stage", NULL);
-    psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0);
-    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label",    0, "search by label", NULL);
-    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label",    0, "define new label", NULL);
-    psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false);
-    psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
-    psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    // -startover
+    psMetadata *startoverArgs = psMetadataAlloc();
+    psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group",    0, "define data_group", NULL);
+    psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0);
+    psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage",    0, "search by stage", NULL);
+    psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0);
+    psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-label",    0, "search by label", NULL);
+    psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-set_label",    0, "define new label", NULL);
+    psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false);
+    psMetadataAddU64(startoverArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
+    psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
     // -revertrun
@@ -105 +106 @@
     psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0);
     psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-state",    0, "define state", NULL);
-    psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label",    0, "define label", NULL);
+    psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
     psMetadataAddBool(revertrunArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);
@@ -151 +152 @@
     psMetadata *processedcomponentArgs = psMetadataAlloc();
     psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(processedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
     psMetadataAddU64(processedcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
     psMetadataAddBool(processedcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -updateprocessedcomponentArgs
+    psMetadata *updateprocessedcomponentArgs = psMetadataAlloc();
+    psMetadataAddS64(updateprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL);
+    psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL);
+    psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL);
+    psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0);
+    psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL);
+    psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
+    psMetadataAddBool(updateprocessedcomponentArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false);
 
     // -toadvance
@@ -180 +193 @@
     psMetadata *updatefilesetArgs = psMetadataAlloc();
     psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0);
-//    psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL);
     psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
@@ -343 +356 @@
     PXOPT_ADD_MODE("-definebyquery",    "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
     PXOPT_ADD_MODE("-updaterun",    "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs);
-    PXOPT_ADD_MODE("-rerun",        "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs);
+    PXOPT_ADD_MODE("-startover",        "reprocess a cleaned distribution run", DISTTOOL_MODE_STARTOVER, startoverArgs);
     PXOPT_ADD_MODE("-revertrun",    "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs);
     PXOPT_ADD_MODE("-pendingcomponent",   "", DISTTOOL_MODE_PENDINGCOMPONENT,    pendingcomponentArgs);
     PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs);
+    PXOPT_ADD_MODE("-updateprocessedcomponent", "", DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentArgs);
     PXOPT_ADD_MODE("-revertcomponent",    "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs);
     PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs);

branches/meh_branches/ppsub_test/ippTools/src/faketool.c

@@ -335 +335 @@
     PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<");
     PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "fakeRun.state", "==");
     PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "==");
     PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "==");
@@ -625 +626 @@
     psMetadata *where = psMetadataAlloc();
     pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "==");
-    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
-    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
-    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-inst", "rawExp..camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "==");
     PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");

branches/meh_branches/ppsub_test/ippTools/src/flatcorr.c

@@ -666 +666 @@
                 "cam",
                 row->cam_id,
+                0,
                 row->workdir,
                 row->reduction,

branches/meh_branches/ppsub_test/ippTools/src/laptool.c

@@ -30 +30 @@
 static bool updateexpMode(pxConfig *config);
 
+static bool diffcheckMode(pxConfig *config);
+
 static bool inactiveexpMode(pxConfig *config);
 
@@ -62 +64 @@
     MODECASE(LAPTOOL_MODE_UPDATEEXP,     updateexpMode);
 
+    MODECASE(LAPTOOL_MODE_DIFFCHECK,     diffcheckMode);
+    
     MODECASE(LAPTOOL_MODE_INACTIVEEXP,   inactiveexpMode);
   default:
@@ -276 +280 @@
   PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "==");
 
+  psMetadata *chipWhere = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, chipWhere, "-seq_id", "lapRun.seq_id", "==");
+  
   // This seems unnecessarily clunky.
   if (!all_obsmode) {
@@ -296 +303 @@
   }
 
+  psString chipWhereClause = psDBGenerateWhereConditionSQL(chipWhere,NULL);
   if (whereClause) {
     psStringSubstitute(&query,whereClause,"@WHERE@");
   }
+  if (chipWhereClause) {
+    psStringSubstitute(&query,chipWhereClause,"@CHIPWHERE@");
+  }
   psFree(where);
+  psFree(chipWhere);
 
   // Fetch exposures
@@ -755 +767 @@
 }
 
-
-    
+static bool diffcheckMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-lap_id", "lapRun.lap_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-exp_id",  "exp_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-seq_id",  "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+
+  psString query = pxDataGet("laptool_WSdiff_check.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return(false);
+  }
+  
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringPrepend(&whereClause, "\n AND ");
+    psStringSubstitute(&query,whereClause,"@WHERE@");
+    psFree(whereClause);
+  }
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error %d",err);
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "lapRunWarpStackSkycells", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+  return(true);
+}    
 
 static bool inactiveexpMode(pxConfig *config)

branches/meh_branches/ppsub_test/ippTools/src/laptool.h

@@ -19 +19 @@
   LAPTOOL_MODE_STACKS,
   LAPTOOL_MODE_UPDATEEXP,
+  LAPTOOL_MODE_DIFFCHECK,
   LAPTOOL_MODE_INACTIVEEXP
 } laptoolMode;

branches/meh_branches/ppsub_test/ippTools/src/laptoolConfig.c

@@ -132 +132 @@
   ADD_OPT(Bool,updateexpArgs, "-inactive",                    "set this exposure to active", 0);
 
+  // -diffcheck
+  psMetadata *diffcheckArgs = psMetadataAlloc();
+  ADD_OPT(S64, diffcheckArgs, "-lap_id",                      "search by lap run ID", 0);
+  ADD_OPT(S64, diffcheckArgs, "-exp_id",                      "search by exposure ID", 0);
+  ADD_OPT(S64, diffcheckArgs, "-chip_id",                     "search by chip ID", 0);
+  ADD_OPT(S64, diffcheckArgs, "-warp_id",                     "search by warp ID", 0);
+  ADD_OPT(S64, diffcheckArgs, "-seq_id",                      "search by lap sequence ID", 0);
+  ADD_OPT(Str, diffcheckArgs, "-skycell_id",                  "search by skycell ID", 0);
+  ADD_OPT(Bool, diffcheckArgs, "-simple",                     "use simple output format", 0);
+  
   // -inactiveexp
   psMetadata *inactiveexpArgs = psMetadataAlloc();
@@ -152 +162 @@
   PXOPT_ADD_MODE("-stacks",                  "", LAPTOOL_MODE_STACKS,           stacksArgs);
   PXOPT_ADD_MODE("-updateexp",               "", LAPTOOL_MODE_UPDATEEXP,        updateexpArgs);
+  PXOPT_ADD_MODE("-diffcheck",               "", LAPTOOL_MODE_DIFFCHECK,        diffcheckArgs);
   PXOPT_ADD_MODE("-inactiveexp",             "", LAPTOOL_MODE_INACTIVEEXP,      inactiveexpArgs);
   

branches/meh_branches/ppsub_test/ippTools/src/magicdstool.c

@@ -53 +53 @@
 
 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
+static bool validDSRunState(const char *state);
 
 # define MODECASE(caseName, func) \
@@ -350 +351 @@
     PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", true, false);
 
     // Optional
     PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
     PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false);
-    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false);
     PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
@@ -374 +375 @@
     pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label
     psString labelName = NULL;                                                    // Name of label
-    psStringAppend(&labelName, "%sRun.label", stage);
+    if (strcmp(stage, "camera")) {
+        psStringAppend(&labelName, "%sRun.label", stage);
+    } else {
+        psStringAppend(&labelName, "camRun.label");
+    }
     pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label
     psFree(labelName);
@@ -392 +397 @@
         break;
       case IPP_STAGE_CAMERA:
+        query = pxDataGet("magicdstool_definecopy_camera.sql");
+        break;
       case IPP_STAGE_DIFF:
       case IPP_STAGE_FAKE:
@@ -1206 +1213 @@
 
     psMetadata *where = psMetadataAlloc();
-    // new state (required
-    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false);
+    // new state
+    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false);
     // old state (required)
     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
@@ -1231 +1238 @@
         if (!strcmp(state, "failed_revert")) {
             new_state = "new";
+        } else if (!strcmp(state, "failed_revert_ud")) {
+            new_state = "update";
         } else if (!strcmp(state, "failed_cleanup")) {
             new_state = "goto_cleaned";
@@ -1237 +1246 @@
             return false;
         }
+    } else {
+        if (!validDSRunState(new_state)) {
+            psError(PS_ERR_UNKNOWN, true, "unexpected value for new state: %s", new_state);
+            return false;
+        }
     }
     if (!p_psDBRunQueryF(config->dbh, query, new_state)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to revert");
+        psError(PS_ERR_UNKNOWN, false, "failed to clear state faults");
         psFree(query);
         return false;
@@ -1397 +1411 @@
           (strcmp(state, "wait") == 0) ||
           (strcmp(state, "failed_revert") == 0) ||
+          (strcmp(state, "failed_revert_ud") == 0) ||
           (strcmp(state, "error_cleaned") == 0) ||
           (strcmp(state, "restored") == 0) ||

branches/meh_branches/ppsub_test/ippTools/src/magicdstoolConfig.c

@@ -79 +79 @@
     psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
     psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false);
-    psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label",    0, "define label", NULL);
+    psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label",    0, "define label (required)", NULL);
     psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL);
     psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL);
@@ -163 +163 @@
     // -clearstatefaults
     psMetadata *clearstatefaultsArgs = psMetadataAlloc();
-    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL);
+    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
     psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL);
     psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);

branches/meh_branches/ppsub_test/ippTools/src/magictool.c

@@ -131 +131 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
-
+    PXOPT_LOOKUP_BOOL(multidiff, config->args, "-multidiff", false);
     psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
     PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
@@ -142 +142 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select.sql");
+        psString query;
+        if (multidiff) {
+          query = pxDataGet("magictool_definebyquery_select_multidiff.sql");
+        }
+        else {
+          query = pxDataGet("magictool_definebyquery_select.sql");
+        }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

branches/meh_branches/ppsub_test/ippTools/src/magictoolConfig.c

@@ -61 +61 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-multidiff", 0, "allow multiple diffs to be magicked with the same label", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);

branches/meh_branches/ppsub_test/ippTools/src/minidvodbtool.c

@@ -89 +89 @@
     psMetadata *where = psMetadataAlloc();
     pxcamGetSearchArgs (config, where);
-    PXOPT_COPY_S64(config->args, where,  "-minidvodb_id",    "minidvodbRun.minidvodbRun_id", "==");
+    PXOPT_COPY_S64(config->args, where,  "-minidvodb_id",    "minidvodbRun.minidvodb_id", "==");
     pxAddLabelSearchArgs (config, where, "-minidvodb_group",     "minidvodbRun.minidvodb_group", "=="); // define using camRun label
    

branches/meh_branches/ppsub_test/ippTools/src/pstamptool.c

@@ -56 +56 @@
 static bool revertdependentMode(pxConfig *config);
 static bool getwebrequestnumMode(pxConfig *config);
+static bool addfileMode(pxConfig *config);
+static bool listfileMode(pxConfig *config);
+static bool deletefileMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -103 +106 @@
         MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode);
         MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode);
+        MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode);
+        MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode);
+        MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -173 +179 @@
 
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, " %s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -1476 +1482 @@
     return true;
 }
+
+static bool addfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+    PXOPT_LOOKUP_STR(path,   config->args, "-path",   true, false);
+
+    if (!pstampFileInsert(config->dbh,
+            0, // file_id
+            job_id,
+            path
+            )) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    if (affected != 1) {
+        psError(PS_ERR_UNKNOWN, false,
+            "should have affected one row but %" PRIu64 " rows were modified",
+            affected);
+        return false;
+    }
+
+    psS64 file_id = psDBLastInsertID(config->dbh);
+    printf("%" PRId64 "\n", file_id);
+
+    return true;
+}
+
+static bool listfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    if (!psListLength(where->list)) {
+        fprintf(stderr, "search arguments are required\n");
+        exit (1);
+    }
+
+    psString query = pxDataGet("pstamptool_listfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool deletefileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+
+    psString query = NULL; 
+    psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected);
+
+    return true;
+}

branches/meh_branches/ppsub_test/ippTools/src/pstamptool.h

@@ -50 +50 @@
     PSTAMPTOOL_MODE_REVERTDEPENDENT,
     PSTAMPTOOL_MODE_GETWEBREQUESTNUM,
+    PSTAMPTOOL_MODE_ADDFILE,
+    PSTAMPTOOL_MODE_LISTFILE,
+    PSTAMPTOOL_MODE_DELETEFILE,
 } pstamptoolMode;
 

branches/meh_branches/ppsub_test/ippTools/src/pstamptoolConfig.c

@@ -274 +274 @@
     psMetadata *modes = psMetadataAlloc();
 
+    // -addfile
+    psMetadata *addfileArgs = psMetadataAlloc();
+    psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id",       0, "define job ID for file (required)", 0);
+    psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path",         0, "define path for file (required)", NULL);
+
+    // -listfile
+    psMetadata *listfileArgs = psMetadataAlloc();
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id",     0, "select by file ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id",      0, "select by request ID", 0);
+    psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple",     0, "use the simple output format", false);
+
+    // -deletefile
+    psMetadata *deletefileArgs = psMetadataAlloc();
+    psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID (required)", 0);
+
     PXOPT_ADD_MODE("-addreq",          "", PSTAMPTOOL_MODE_ADDREQ,       addreqArgs);
     PXOPT_ADD_MODE("-pendingreq",      "", PSTAMPTOOL_MODE_PENDINGREQ,   pendingreqArgs);
@@ -302 +319 @@
     PXOPT_ADD_MODE("-project",         "", PSTAMPTOOL_MODE_PROJECT,    projectArgs);
     PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM,   getwebrequestnumArgs);
+    PXOPT_ADD_MODE("-addfile",         "", PSTAMPTOOL_MODE_ADDFILE,   addfileArgs);
+    PXOPT_ADD_MODE("-listfile",        "", PSTAMPTOOL_MODE_LISTFILE,   listfileArgs);
+    PXOPT_ADD_MODE("-deletefile",      "", PSTAMPTOOL_MODE_DELETEFILE,   deletefileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/meh_branches/ppsub_test/ippTools/src/pxadd.c

@@ -131 +131 @@
                        char *stage,
                        psS64 stage_id,
+                       psS32 stage_extra1,
                        char *workdir,
                        char *reduction,
@@ -159 +160 @@
         psMemSetPersistent(query, true);
       }
+      if (strcmp(stage,"staticsky_multi") == 0) {
+        query = pxDataGet("addtool_queue_sky_id_multi.sql");
+        psMemSetPersistent(query, true);
+      }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -170 +175 @@
     // on 32/64
     if (!p_psDBRunQueryF(config->dbh, query,
+                         stage_extra1 ? stage_extra1 : 0,
                          "new", // state
                          workdir  ? workdir   : "NULL",

branches/meh_branches/ppsub_test/ippTools/src/pxadd.h

@@ -33 +33 @@
                        char *stage,
                        psS64 stage_id,
+                       psS32 stage_extra1,
                        char *workdir,
                        char *reduction,

branches/meh_branches/ppsub_test/ippTools/src/pxtools.c

@@ -35 +35 @@
     if (!strcmp(state, "drop")) return true;
     if (!strcmp(state, "wait")) return true;
+    if (!strcmp(state, "keep")) return true;
     if (!strcmp(state, "goto_cleaned")) return true;
     if (!strcmp(state, "error_cleaned")) return true;

branches/meh_branches/ppsub_test/ippTools/src/regtool.c

@@ -268 +268 @@
       }
     }
+    char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a');
+    if (is_stareim) {
+      continue;
+    }
     
     if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
@@ -386 +390 @@
       }
     }
+    char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a');
+    if (is_stareim) {
+      continue;
+    }
+
 
     bool status = false;
@@ -882 +891 @@
     PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false);
     PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false);
+    PXOPT_LOOKUP_BOOL(set_ignored, config->args, "-set_ignored", false);
+    PXOPT_LOOKUP_BOOL(clear_ignored, config->args, "-clear_ignored", false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     
-    if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) {
-        psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected");
+    if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state && !set_ignored && !clear_ignored) {
+        psError(PS_ERR_UNKNOWN, false, "at least one of -fault or -burntool_state -set_ignored -clear_ignored or -set_state must be selected");
         return false;
     }
     if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) {
         psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected");
+        return false;
+    }
+    if (set_ignored && clear_ignored) {
+        psError(PS_ERR_UNKNOWN, true, "only one of -set_ignored or -clear_ignored may be selected");
         return false;
     }
@@ -898 +913 @@
 
     if (fault != INT16_MAX) {
-        // this is fairly dangerous : can set all if the where is not set...
+        // this is fairly dangerous : can set all if the where is not set... but since exp_id and class_id
+        // were required above this is safe
         if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault, 0)) {
             psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
@@ -930 +946 @@
       }
       psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state);
+    }
+    if (set_ignored) {
+      if (setvalues) {
+        psStringAppend(&setvalues,",");
+      }
+      psStringAppend(&setvalues,"rawImfile.ignored = 1");
+    }
+    if (clear_ignored) {
+      if (setvalues) {
+        psStringAppend(&setvalues,",");
+      }
+      psStringAppend(&setvalues,"rawImfile.ignored = 0");
     }
     if (hostname) {

branches/meh_branches/ppsub_test/ippTools/src/regtoolConfig.c

@@ -194 +194 @@
     ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum",     "set md5sum", NULL);
     ADD_OPT(Str, updateprocessedimfileArgs, "-set_state",      "set data state", NULL);
+    psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_ignored",  0,        "set imfile to be ignored for processing", false);
+    psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-clear_ignored",  0,        "set imfile to not be ignored for processing", false);
 
     

branches/meh_branches/ppsub_test/ippTools/src/staticskytool.c

@@ -38 +38 @@
 static bool resultMode(pxConfig *config);
 static bool revertMode(pxConfig *config);
-static bool updateresult(pxConfig *config);
+static bool updateresultMode(pxConfig *config);
 static bool exportrunMode(pxConfig *config);
 static bool importrunMode(pxConfig *config);
+
+static bool defineskycalrunMode(pxConfig *config);
+static bool updateskycalrunMode(pxConfig *config);
+static bool pendingskycalrunMode(pxConfig *config);
+static bool addskycalresultMode(pxConfig *config);
+static bool skycalresultMode(pxConfig *config);
+static bool revertskycalresultMode(pxConfig *config);
+static bool updateskycalresultMode(pxConfig *config);
 
 static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state);
@@ -69 +77 @@
         MODECASE(STATICSKYTOOL_MODE_RESULT,            resultMode);
         MODECASE(STATICSKYTOOL_MODE_REVERT,            revertMode);
-        MODECASE(STATICSKYTOOL_MODE_UPDATERESULT,      updateresult);
+        MODECASE(STATICSKYTOOL_MODE_UPDATERESULT,      updateresultMode);
         MODECASE(STATICSKYTOOL_MODE_EXPORTRUN,         exportrunMode);
         MODECASE(STATICSKYTOOL_MODE_IMPORTRUN,         importrunMode);
+        MODECASE(STATICSKYTOOL_MODE_DEFINESKYCALRUN,   defineskycalrunMode);
+        MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRUN,   updateskycalrunMode);
+        MODECASE(STATICSKYTOOL_MODE_PENDINGSKYCALRUN,  pendingskycalrunMode);
+        MODECASE(STATICSKYTOOL_MODE_ADDSKYCALRESULT,   addskycalresultMode);
+        MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRESULT,updateskycalresultMode);
+        MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode);
+        MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT,      skycalresultMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -97 +112 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // required options
-    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false);
+    PXOPT_LOOKUP_STR(workdir,     config->args, "-set_workdir", true, false);
+    PXOPT_LOOKUP_STR(label,       config->args, "-set_label", true, false);
 
     // optional
-    PXOPT_LOOKUP_STR(label,       config->args, "-set_label", true, false);
     PXOPT_LOOKUP_STR(data_group,  config->args, "-set_data_group", false, false);
     PXOPT_LOOKUP_STR(dist_group,  config->args, "-set_dist_group", false, false);
@@ -111 +124 @@
     psMetadata *whereMD = psMetadataAlloc();
 
-    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "==");
+    PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id",      "stackRun.stack_id",         "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "LIKE");
     PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id",       "stackRun.tess_id",          "==");
     PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">=");
@@ -735 +749 @@
 }
 
-static bool updateresult(pxConfig *config)
+static bool updateresultMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -978 +992 @@
 # endif
 
+static bool defineskycalrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required options
+    // none required. We get workdir, etc from staticskyRun if not provided
+
+    // optional
+    PXOPT_LOOKUP_STR(label,       config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(workdir,     config->args, "-set_workdir", false, false);
+    PXOPT_LOOKUP_STR(data_group,  config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group,  config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(note,        config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
+
+    psMetadata *whereMD = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id",        "staticskyRun.sky_id",       "==");
+    PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id",      "stackRun.stack_id",         "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id",       "stackRun.tess_id",          "==");
+    PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">=");
+    pxAddLabelSearchArgs(config, whereMD, "-select_label",         "stackRun.label",            "LIKE");
+    pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "stackRun.data_group",       "LIKE");
+    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
+
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psString query = pxDataGet("staticskytool_defineskycalrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(whereMD);
+        return false;
+    }
+
+    if (!psListLength(whereMD->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        psFree(whereMD);
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL);
+    psStringAppend(&query, "\nAND %s", whereClause);
+    psFree(whereClause);
+    psFree(whereMD);
+
+    if (!rerun) {
+        if (label)  {
+            psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label);
+        }
+        psStringAppend(&query, "\nAND skycal_id IS NULL");
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs
+    // we now need to loop over all of these and for each pair, select the best set of
+    // inputs
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psWarning("staticskytool: no rows found");
+        psFree(output);
+        return true;
+    }
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "skycalRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < output->n; i++) {
+        psMetadata *row = output->data[i]; // Row from select
+        bool status;
+
+        psS64 sky_id = psMetadataLookupS64(&status, row, "sky_id");
+        psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id");
+        psString sky_workdir = psMetadataLookupStr(&status, row, "workdir");
+        psString sky_label =  psMetadataLookupStr(&status, row, "label");
+        psString sky_data_group =  psMetadataLookupStr(&status, row, "data_group");
+
+        // create a staticskyRun
+        if (!skycalRunInsert(config->dbh,
+                                0x0,         // skycal_id
+                                sky_id,
+                                stack_id,
+                                "new",       // state
+                                workdir ? workdir : sky_workdir,
+                                label ? label : sky_label,
+                                data_group ? data_group : sky_data_group,
+                                dist_group,
+                                reduction,
+                                registered,
+                                note
+                )
+            ) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            return false;
+        }
+
+    }
+    psFree(output);
+    return true;
+}
+
+static bool updateskycalrunMode(pxConfig *config)
+{
+#ifdef notyet
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-sky_id",  "sky_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-label",   "label",    "==");
+    PXOPT_COPY_STR(config->args, where, "-state",   "state",    "==");
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE staticskyRun");
+
+    // pxUpdateRun gets parameters from config->args and updates
+    bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false);
+    psFree(query);
+    psFree(where);
+
+    return result;
+#endif
+    return false;
+}
+static bool pendingskycalrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *whereMD = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, whereMD, "-skycal_id", "skycal_id", "==");
+    PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "==");
+    pxAddLabelSearchArgs (config, whereMD, "-label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-filter", "stackRun.filter", "LIKE");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("staticskytool_pendingskycalrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!psListLength(whereMD->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        psFree(whereMD);
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL);
+    psStringAppend(&query, "\n AND %s", whereClause);
+    psFree(whereClause);
+    psFree(whereMD);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, "\n%s", limitString);
+        psFree(limitString);
+    }
+
+    // the where clause is required and matches the WHERE hook format string
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("staticskytool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "pendingskycalRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+    return true;
+}
+
+static bool addskycalresultMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(skycal_id, config->args, "-skycal_id", true, false);
+
+    // optional
+    PXOPT_LOOKUP_F32(sigma_ra, config->args,  "-sigma_ra", false, false);
+    PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false);
+
+    PXOPT_LOOKUP_F32(zpt_obs, config->args,   "-zpt_obs", false, false);
+    PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false);
+
+
+    PXOPT_LOOKUP_F32(dtime_script, config->args,   "-dtime_script", false, false);
+    PXOPT_LOOKUP_F32(dtime_astrom, config->args,   "-dtime_astrom", false, false);
+
+    PXOPT_LOOKUP_STR(hostname, config->args,    "-hostname", false, false);
+//    PXOPT_LOOKUP_S32(n_stars, config->args,     "-n_stars", false, false);
+    PXOPT_LOOKUP_S32(n_astrom, config->args,    "-n_astrom", false, false);
+
+    PXOPT_LOOKUP_STR(path_base, config->args,   "-path_base", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args,       "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args,     "-quality", false, false);
+
+    PXOPT_LOOKUP_STR(ver_pslib, config->args,   "-ver_pslib", false, false);
+    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
+    PXOPT_LOOKUP_STR(ver_psphot, config->args,  "-ver_psphot", false, false);
+    PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false);
+    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
+
+    psString software_ver = NULL;
+    if ((ver_pslib)&&(ver_psmodules)) {
+      software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules);
+    }
+    if (ver_psphot) {
+      software_ver = pxMergeCodeVersions(software_ver,ver_psphot);
+    }
+    if (ver_psastro) {
+      software_ver = pxMergeCodeVersions(software_ver,ver_psastro);
+    }
+    if (ver_ppstats) {
+      software_ver = pxMergeCodeVersions(software_ver,ver_ppstats);
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    skycalResultRow *row = skycalResultRowAlloc(
+        skycal_id,
+        path_base,
+        dtime_script,
+        dtime_astrom,
+        sigma_ra,
+        sigma_dec,
+        n_astrom,
+        zpt_obs,
+        zpt_stdev,
+        quality,
+        software_ver,
+        hostname,
+        fault
+        );
+
+    if (!skycalResultInsertObject(config->dbh, row)) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(row);
+        return false;
+    }
+
+    psFree(row);
+
+    if (fault) {
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        return true;
+    }
+
+    psString query = "UPDATE skycalRun SET state = 'full' WHERE skycal_id = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, skycal_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+static bool updateskycalresultMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", true, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",   "skycal_id",   "==");
+
+    if (!pxSetFaultCode(config->dbh, "skycalResult", where, fault, quality)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        psFree (where);
+        return false;
+    }
+    psFree (where);
+    return true;
+}
+static bool skycalresultMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-sky_id",     "skycalRun.sky_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",   "skycalRun.stack_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter",     "stackRun.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-label",      "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "skycalResult.fault", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("staticskytool_skycalresult.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("staticskytool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+static bool revertskycalresultMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-skycal_id", "staticskyResult.sky_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "staticskyRun.label", "==");
+    pxAddLabelSearchArgs(config, where, "-data_group", "staticskyRun.data_group", "==");
+    pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    // Delete product
+    psString delete = pxDataGet("staticskytool_revertskycal.sql");
+    if (!delete) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&delete, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, delete)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(delete);
+        psFree(where);
+        return false;
+    }
+    psFree(delete);
+
+    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
+    psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows);
+
+    psFree(where);
+    return true;
+}

branches/meh_branches/ppsub_test/ippTools/src/staticskytool.h

@@ -34 +34 @@
     STATICSKYTOOL_MODE_UPDATERESULT,
     STATICSKYTOOL_MODE_EXPORTRUN,
-    STATICSKYTOOL_MODE_IMPORTRUN
+    STATICSKYTOOL_MODE_IMPORTRUN,
+
+    STATICSKYTOOL_MODE_DEFINESKYCALRUN,
+    STATICSKYTOOL_MODE_PENDINGSKYCALRUN,
+    STATICSKYTOOL_MODE_UPDATESKYCALRUN,
+    STATICSKYTOOL_MODE_ADDSKYCALRESULT,
+    STATICSKYTOOL_MODE_UPDATESKYCALRESULT,
+    STATICSKYTOOL_MODE_SKYCALRESULT,
+    STATICSKYTOOL_MODE_REVERTSKYCALRESULT,
 } staticskytoolMode;
 

branches/meh_branches/ppsub_test/ippTools/src/staticskytoolConfig.c

@@ -48 +48 @@
     // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries
     psMetadata *definebyqueryArgs = psMetadataAlloc();
+    psMetadataAddS64(definebyqueryArgs,  PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
@@ -138 +139 @@
     psMetadata *importrunArgs = psMetadataAlloc();
     psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL);
-
+  
+    // -defineskycalrun
+    psMetadata *defineskycalrunArgs = psMetadataAlloc();
+    psMetadataAddS64(defineskycalrunArgs,  PS_LIST_TAIL, "-select_sky_id", 0, "search for sky_id", 0);
+    psMetadataAddS64(defineskycalrunArgs,  PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
+    psMetadataAddF32(defineskycalrunArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_note", 0, "define note", NULL);
+    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL);
+    psMetadataAddTime(defineskycalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-rerun",  0, "queue new run even if one already exists for inputs", false);
+    psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-pretend",  0, "do not actually modify the database", false);
+    psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs",  0, "list inputs, do not modify database", false);
+    psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -updateskycalrun
+    psMetadata *updateskycalrunArgs = psMetadataAlloc();
+    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL);
+
+    // -pendingskycalrun
+    psMetadata *pendingskycalrunArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
+    psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0);
+    psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddU64(pendingskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(pendingskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+
+    // -addskycalresult
+    psMetadata *addskycalresultArgs = psMetadataAlloc();
+    psMetadataAddS64(addskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0,            "define camtool ID (required)", 0);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0,            "define exposure E ra", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0,            "define exposure E dec", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0,   "define observed zero point", 0);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0,   "define observed zero point stdandard deviation", 0);
+
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);
+    psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN);
+
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", NULL);
+    // psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_stars", 0,            "define number of stars", 0);
+    psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_astrom", 0,            "define number of astrometry reference objects", 0);
+
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);
+    psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
+
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL);
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL);
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL);
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL);
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL);
+    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL);
+
+    // -revertskycal
+    psMetadata *revertskycalArgs= psMetadataAlloc();
+    psMetadataAddS64(revertskycalArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by staticsky ID", 0);
+    psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
+    psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0);
+    psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0);
+    psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false);
+
+    // -updateskycalresult
+    psMetadata *updateskycalresultArgs = psMetadataAlloc();
+    psMetadataAddS64(updateskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define staticksky ID (required)", 0);
+    psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code (required)", 0);
+    psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality code", 0);
+
+    // -skycalresult
+    psMetadata *skycalresultArgs= psMetadataAlloc();
+    psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
+    psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0);
+    psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0);
+    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0);
+    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0);
+    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0);
+    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL);
+    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
+    psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0);
+    psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
     psFree(now);
@@ -155 +253 @@
     PXOPT_ADD_MODE("-exportrun",     "Export run",           STATICSKYTOOL_MODE_EXPORTRUN,     exportrunArgs);
     PXOPT_ADD_MODE("-importrun",     "Import run",           STATICSKYTOOL_MODE_IMPORTRUN,     importrunArgs);
+    PXOPT_ADD_MODE("-defineskycalrun", "Define a new skycalrun", STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunArgs);
+    PXOPT_ADD_MODE("-updateskycalrun", "Update a skycalrun", STATICSKYTOOL_MODE_UPDATESKYCALRUN,     updateskycalrunArgs);
+    PXOPT_ADD_MODE("-pendingskycalrun", "Get skcal runs to do",       STATICSKYTOOL_MODE_PENDINGSKYCALRUN,          pendingskycalrunArgs);
+    PXOPT_ADD_MODE("-addskycalresult", "add skycal result",       STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultArgs);
+    PXOPT_ADD_MODE("-revertskycal",  "revert faulted skycal run", STATICSKYTOOL_MODE_REVERTSKYCALRESULT, revertskycalArgs);
+    PXOPT_ADD_MODE("-updateskycal",  "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs);
+    PXOPT_ADD_MODE("-skycalresult",  "Get result of skycal run",  STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/meh_branches/ppsub_test/ippTools/src/warptool.c

@@ -975 +975 @@
     psFree(where);
 
+    psArray *allOutput = psArrayAllocEmpty(0);
     // Now loop over the labels and query each individually
     for (long i = 0; i < psArrayLength(labelOutput); i++) {
@@ -983 +984 @@
 
         psString query = psStringCopy(originalQuery);
-
+        psStringAppend(&query,"\nORDER BY warp_id");
         // treat limit == 0 as "no limit"
         psString limitString = psStringCopy("");
@@ -992 +993 @@
             psStringAppend(&query, "%s", limitString);
         }
+
         psString thisWhere = NULL;
         if (whereStr) {
@@ -1023 +1025 @@
         long outputLength = psArrayLength(output);
         if (outputLength) {
-            // negative simple so the default is true
-            if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to print array");
-                psFree(output);
-                return false;
+            for (int i = 0; i< outputLength; i++) {
+                psPtr ptr = output->data[i];
+                allOutput = psArrayAdd(allOutput, outputLength - i + 10, ptr);
+                if (!allOutput) {
+                    psError(PS_ERR_UNKNOWN, false, "failed to add element to array");
+                    return false;
+                }
             }
         } else {
@@ -1044 +1048 @@
         }
     }
+    long allLabelsLength = psArrayLength(allOutput);
+    if (allLabelsLength) {
+        if (!ippdbPrintMetadatas(stdout, allOutput, "warpPendingSkyCell", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(allOutput);
+            return false;
+        }
+    }
+    psFree(allOutput);
     psFree(labelOutput);
     psFree(whereStr);
@@ -2039 +2052 @@
         // magicked is only an argument for for -tofullskyfile
         PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+        PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
         if (magicked) {
             psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked);
             psStringAppend(&set_magicked_run, "\n,  warpRun.magicked = %" PRId64, magicked);
         }
+        if (quality) {
+          psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.quality = %"PRId16, quality);
+        }
     } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
         // if magicked is currently nonzero set it to -1
@@ -2421 +2438 @@
     PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",    "chipRun.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",     "camRun.cam_id", "==");
     PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
     PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");

branches/meh_branches/ppsub_test/ippTools/src/warptoolConfig.c

@@ -269 +269 @@
     psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id",  0,          "search by tessellation ID", NULL);
     psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
-    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure tag", 0);
-    psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", 0);
-    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag", 0);
+    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure id", 0);
+    psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", NULL);
+    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0,           "search by chip id", 0);
+    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-cam_id",  0,           "search by cam id", 0);
+    psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag (fake_id)", 0);
     psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
     psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
@@ -396 +398 @@
     psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
     psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked",  0, "new value for magicked", 0);
-
+    psMetadataAddS16(tofullskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "new value for quality", 0);
+    
     // -updateskyfile
     psMetadata *updateskyfileArgs = psMetadataAlloc();

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool