Changeset 35308 for trunk/ippTools/src/laptool.c

Timestamp:

Mar 12, 2013, 3:29:45 PM (13 years ago)

Author:

bills

Message:

sript, ippTools, and database changes to support queuing staticsky runs
on the basis of lapGroups collections of lapRuns that have finished for
each of a list of filters

File:

• trunk/ippTools/src/laptool.c (modified) (3 diffs)

trunk/ippTools/src/laptool.c

@@ -34 +34 @@
 static bool inactiveexpMode(pxConfig *config);
 
+// Groups
+static bool definegroupMode(pxConfig *config);
+static bool pendinggroupMode(pxConfig *config);
+static bool filtersforgroupMode(pxConfig *config);
+static bool updategroupMode(pxConfig *config);
+static bool revertgroupMode(pxConfig *config);
+static bool listgroupMode(pxConfig *config);
+
 # define MODECASE(caseName, func) \
   case caseName: \
@@ -67 +75 @@
     
     MODECASE(LAPTOOL_MODE_INACTIVEEXP,   inactiveexpMode);
+
+    MODECASE(LAPTOOL_MODE_DEFINEGROUP,   definegroupMode);
+    MODECASE(LAPTOOL_MODE_PENDINGGROUP,  pendinggroupMode);
+    MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode);
+    MODECASE(LAPTOOL_MODE_UPDATEGROUP,   updategroupMode);
+    MODECASE(LAPTOOL_MODE_REVERTGROUP,   revertgroupMode);
+    MODECASE(LAPTOOL_MODE_LISTGROUP,     listgroupMode);
+
   default:
     psAbort("invalid option (this should not happen)");
@@ -900 +916 @@
   return(true);
 }
-
+// ---------------------------
+// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters
+
+static bool definegroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_S64(seq_id,          config->args, "-seq_id",    true, false);
+  PXOPT_LOOKUP_STR(label,           config->args, "-set_label", true, false);
+
+  PXOPT_LOOKUP_BOOL(pretend,        config->args, "-pretend",    false);
+  PXOPT_LOOKUP_BOOL(simple,         config->args, "-simple",    false);
+  PXOPT_LOOKUP_U64(limit,           config->args, "-limit",     false, false);
+
+  // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs)
+  psMetadataItem *item = psMetadataLookup(config->args, "-filter");
+  psAssert (item, "-filter argument not found in config->args");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter");
+  psAssert (item->data.list->n, "%s should at least have a place-holder", "filter");
+  psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data;
+  psAssert (entry, "%s should at least have a place-holder", "filter");
+  // end of checking
+
+  // Now if the ony entry is the place-holder then the user supplied no -filter arguments
+  // which are required
+  if (!entry->data.str) {
+    psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required");
+    return false;
+  }
+
+  int nFilters = item->data.list->n;
+
+  psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n");
+
+  // We construct a query joining completed lap runs with the same lapSequence.seq_id,
+  // tess_id, and projection_cell 
+
+  // select the lap_ids for the various filters. These are only used for debugging.
+  psStringAppend(&query, "lap_id_%d", 0);
+  for (int i = 1; i < nFilters; i++) {
+    psStringAppend(&query, ", lap_id_%d", i);
+  }
+
+
+  // sub query for each supplied filter
+  char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n"
+    "FROM lapRun\n"
+    "WHERE filter LIKE '%s'\n"
+    "   AND lapRun.seq_id = %"PRId64"\n"
+    "   AND (lapRun.state = 'done' or lapRun.state = 'full')\n"
+    " ) as lap_%d\n";
+
+  // loop over supplied filters and flesh out the query using the format above
+  psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true);
+  psMetadataItem *filterItem = NULL;
+  int i = -1;
+  while ((filterItem = psListGetAndIncrement(iter))) {
+    ++i;
+    if (i == 0) {
+      psStringAppend(&query, "\nFROM\n");
+    } else {
+      psStringAppend(&query, "\nJOIN\n");
+    }
+
+    psString filter = filterItem->data.str;
+    psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i);
+
+    if (i != 0) {
+      psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n");
+    }
+  }
+  // now join to lapGroup
+  psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n");
+
+  // we only want projection cells which do not already of an entry
+  psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n");
+  psStringAppend(&query, "\nORDER by projection_cell\n");
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, "\n %s", limitString);
+    psFree(limitString);
+  }
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+  }
+
+  if (!psArrayLength(output)) {
+      psTrace("laptool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+  }
+
+  if (pretend) {
+    if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+    return true;
+  }
+
+  if (!psDBTransaction(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psTime *now = psTimeGetNow(PS_TIME_TAI);
+
+  psString last_projection_cell = NULL;
+  for (long i = 0; i < psArrayLength(output); i++) {
+    psMetadata *row = output->data[i];
+
+    psString tess_id = psMetadataLookupStr(NULL, row, "tess_id");
+    psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell");
+    if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) {
+        // duplicate lap runs for a filter will generate multiple rows
+        // Since we care about projection_cells not lapRuns per se this is not problem.
+        // Skip any duplicates.
+        continue;
+    }
+    last_projection_cell = projection_cell;
+
+    lapGroupRow *group = lapGroupRowAlloc(
+                                  seq_id,
+                                  tess_id,
+                                  projection_cell,
+                                  "new",  // state
+                                  label,
+                                  now,    // registered
+                                  0       // fault
+                                  );
+    if (!group) {
+      psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object");
+      psFree(output);
+      psFree(now);
+      return(false);
+    }
+
+
+    if (!lapGroupInsertObject(config->dbh, group)) {
+      if (!psDBRollback(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+      }
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(output);
+      return(true);
+    }
+  }
+
+  // point of no return
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psFree(now);
+  psFree(output);
+
+  return(true);  
+}
+
+static bool pendinggroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "==");
+  
+  psString query = pxDataGet("laptool_pendinggroup.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  }
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, "\n %s", limitString);
+    psFree(limitString);
+  }
+  
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}
+static bool filtersforgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+  
+  psString query = pxDataGet("laptool_filtersforgroup.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " WHERE %s", whereClause);
+  psFree(whereClause);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}
+
+static bool updategroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  // check that all of the keys that define a lapGroup are supplied
+  PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false);
+  PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
+
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+
+  psMetadata *values = psMetadataAlloc();
+  PXOPT_COPY_STR(config->args, values, "-set_state", "state", "==");
+  PXOPT_COPY_STR(config->args, values, "-set_label", "label", "==");
+  PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "==");
+
+  if (!psListLength(values->list)) {
+    psFree(values);
+    psFree(where);
+    psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column");
+    return false;
+  }
+
+  long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values);
+  if (rows < 1) {
+    psFree(values);
+    psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell);
+    return false;
+  }
+  psFree(values);
+
+  return(true);
+}
+
+static bool revertgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psMetadata *where = psMetadataAlloc();
+  pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
+
+  if (!psListLength(where->list)) {
+    psFree(where);
+    psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required");
+    return false;
+  }
+
+  psString query = pxDataGet("laptool_revertgroup.sql");
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " AND %s", whereClause);
+  psFree(whereClause);
+  psFree(where);
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psU64 affected = psDBAffectedRows(config->dbh);
+  psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected);
+
+  return true;
+}
+
+static bool listgroupMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psError(PXTOOLS_ERR_SYS, true, "not yet implemented");
+  return false;
+
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-seq_name",   "name",   "LIKE");
+
+  psString query = pxDataGet("laptool_listsequence.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+  }
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("laptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (psArrayLength(output)) {
+    if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+
+  psFree(output);
+
+  return true;
+}