Changeset 30579 for branches/eam_branches/ipp-20101205/ippTools/src

Timestamp:

Feb 11, 2011, 2:49:34 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20101205/ippTools/src

Files:

• . (modified) (1 prop)
• chiptool.c (modified) (3 diffs)
• difftool.c (modified) (3 diffs)
• disttool.c (modified) (1 diff)
• disttoolConfig.c (modified) (1 diff)
• dqstatstool.c (modified) (6 diffs)
• magicdstool.c (modified) (4 diffs)
• magictool.c (modified) (1 diff, 1 prop)
• magictoolConfig.c (modified) (1 diff)
• pstamptool.c (modified) (2 diffs)
• pstamptoolConfig.c (modified) (1 diff)
• pxtools.h (modified) (1 diff)
• warptool.c (modified) (6 diffs)

branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c

@@ -838 +838 @@
     PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "==");
 
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
     if (!psListLength(where->list)
         && !psMetadataLookupBool(NULL, config->args, "-all")) {
@@ -874 +876 @@
     }
     psFree(query);
+
+    if (!fault) {
+        // If fault has not been supplied, don't revert update faults with the magic value
+        // We don't do this for new runs because then they would never complete
+        // quality should be used to drop bad components
+        psStringAppend(&query_update, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
+    }
     if (!p_psDBRunQuery(config->dbh, query_update)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1919 +1928 @@
     }
 
+    // we do not update components with the magic fault value. They are non-updateable
+    psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
+
     if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");

branches/eam_branches/ipp-20101205/ippTools/src/difftool.c

@@ -917 +917 @@
     PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");
 
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
     if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
@@ -950 +952 @@
         psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh));
 
+        if (!fault) {
+            // If fault has not been supplied, don't revert update faults with the magic value
+            // We don't do this for new runs because then they would never complete
+            // quality should be used to drop bad components
+            psStringAppend(&query_updated, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
+        }
         if (!p_psDBRunQuery(config->dbh, query_updated)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -3201 +3209 @@
         psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id);
     }
+    // we do not update components with the magic fault value. They are non-updateable
+    // (But can be recovered with "difftool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
+    psStringAppend(&query, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
 
     if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) {

branches/eam_branches/ipp-20101205/ippTools/src/disttool.c

@@ -1185 +1185 @@
     PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "rcDSFileset.fault", "==");
 

branches/eam_branches/ipp-20101205/ippTools/src/disttoolConfig.c

@@ -197 +197 @@
     psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0);
     psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-state",   0, "define state", NULL);
-    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label",   0, "define label", NULL);
+    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label",   PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddS16(revertfilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
     psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);

branches/eam_branches/ipp-20101205/ippTools/src/dqstatstool.c

@@ -92 +92 @@
   }
   else {
-    psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) ");
+    psStringAppend(&query, " AND (warp_id IS NOT NULL AND invalid IS NULL) ");
   }
 
@@ -402 +402 @@
 
     if (psMetadataLookupStr(&status,enginedef,"COLNAME")&&
-        isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&&
-        isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) {
+        (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"STRICT") == 0)&&
+        (isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&&
+         isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM")))) {
       psArrayAdd(decisionRules,decisionRules->n,enginedef);
+    }
+    else if (psMetadataLookupStr(&status,enginedef,"COLNAME")&&
+             (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"CDF") == 0)&&
+             (isfinite(psMetadataLookupF32(&status,enginedef,"CDF00"))&&
+              isfinite(psMetadataLookupF32(&status,enginedef,"CDF50"))&&
+              isfinite(psMetadataLookupF32(&status,enginedef,"CDF100")))) {
+      psArrayAdd(decisionRules,decisionRules->n, enginedef);
+    }
+    else if (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"UNUSED") == 0) {
     }
     else {
@@ -637 +647 @@
     // Calculate if this exposure was "good" or not.
     bool accept = true;
+    psF32 quality = 1.0;
     for (long j = 0; j < decisionRules->n; j++) {
       psMetadata *rule = decisionRules->data[j];
@@ -644 +655 @@
         psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name");
         return(false);
-      }      
-      psF32 min_value  = psMetadataLookupF32(&status,rule,"MINIMUM");
-      if (!status) {
-        psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname);
-        return(false);
-      }      
-      psF32 max_value  = psMetadataLookupF32(&status,rule,"MAXIMUM");
-      if (!status) {
-        psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname);
-        return(false);
-      }
-      psString filter = psMetadataLookupStr(&status,rule,"FILTER");
-      if (!status) {
-        status = true;
-      }
-      if (filter) {
-        psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER");
-        if (strcmp(filter,imfilter) != 0) {
-          continue;
-        }
       }
       // Not happy with this being set to a F32. Can this ever be something else?
       psF32 value      = psMetadataLookupF32(&status,tableRow,colname);
       if (!status) {
-        psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value);
+        psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f) in compare with rule.",
+                colname,value);
         psFree(colNames);
         psFree(colTypes);
@@ -677 +669 @@
         return(false);
       }
-      if ((value > max_value)||(value < min_value)) {
-        accept = false;
-      }      
+      psString filter = psMetadataLookupStr(&status,rule,"FILTER");
+      if (!status) {
+        status = true;
+      }
+      if (filter) {
+        psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER");
+        if (strcmp(filter,imfilter) != 0) {
+          continue;
+        }
+      }
+      
+      if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"STRICT") == 0) {
+        psF32 min_value  = psMetadataLookupF32(&status,rule,"MINIMUM");
+        if (!status) {
+          psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname);
+          return(false);
+        }      
+        psF32 max_value  = psMetadataLookupF32(&status,rule,"MAXIMUM");
+        if (!status) {
+          psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname);
+          return(false);
+        }
+
+        if ((value > max_value)||(value < min_value)) {
+          accept = false;
+        }
+      } // End STRICT min/max rule parsing
+      else if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"CDF") == 0) {
+        // I'm just going to implement a quick linear interpolation over the points that exist.
+        psF32 min_value,max_value = 99;
+        psF32 min_cdf = 0.0,max_cdf = 1.0;
+        psF32 cdf;
+        psF32 try[22] = {0.00,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50,
+                         0.55,0.60,0.65,0.70,0.75,0.80,0.85,0.90,0.95,0.99,1.00};
+        int i;
+        int done = 0;
+        status = true;
+        min_value = psMetadataLookupF32(&status,rule,"CDF00");
+        min_cdf   = 0.0;
+        if (!isfinite(value)) {
+          quality = 0.0;
+          continue;
+        }
+        if (!status) {
+          status = true;
+          psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find CDF00 value for %s",colname);
+        }
+        if (value < min_value) {
+          if (quality > min_cdf) {
+            quality = 0.0;
+            continue;
+          }
+        }
+
+        for (i = 1; i < 22; i++) {
+          psString label = psStringCopy("");
+          psStringAppend(&label,"CDF%02d",(int) (100 * try[i]));
+          
+          max_value = psMetadataLookupF32(&status,rule,label);
+          max_cdf   = try[i];
+          if (!status) {
+            status = true;
+            psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find %s value for %s",label,colname);
+          }
+          if ((value < max_value)&&(value > min_value)) {
+            cdf = ((value - min_value) * max_cdf + (max_value - value) * min_cdf) / (max_value - min_value);
+            //      fprintf(stderr,"value: %f %f %f %f %f %f %d %d %f\n",value,quality,min_cdf,min_value,max_cdf,max_value, value < max_value, value > min_value,cdf);
+            if (quality > cdf) {
+              quality = cdf;
+              done = 1;
+              i = 25;
+            }
+          }
+          min_value = max_value;
+          min_cdf = max_cdf;
+        }
+        if (done) {
+          continue;
+        }
+        // Reached end of array without finding the value;
+        if (value > min_value) {
+          if (quality > min_cdf) {
+            quality = 1.0;
+          }
+        }
+
+
+      } // End CDF rule parsing
     }
     bool success;
@@ -695 +772 @@
       return(false);
     }
+    success = psMetadataAddF32(tableRow,PS_LIST_TAIL,"QUALITY",0,"",quality);
+    if (!success) {
+      psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row.");
+      psFree(colNames);
+      psFree(colTypes);
+      psFree(contents);
+      psFree(query);
+      psFree(outTable);
+      psFree(statsoutput);
+      psFree(statIter);
+      psFree(tableRow);
+      return(false);
+    }
 
     // Add the row to the table array.

branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c

@@ -1112 +1112 @@
 
     PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     psString queryFile = NULL;
+    bool stateIsUpdate = false;
     if (state) {
         if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) {
@@ -1119 +1121 @@
         } else if (!strcmp(state, "update")) {
             queryFile = "magicdstool_revertupdated.sql";
+            stateIsUpdate = true;
         } else {
             psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state);
@@ -1141 +1144 @@
     }
     psFree(where);
+
+    if (stateIsUpdate && !fault) {
+        // If fault has not been supplied, don't revert update faults with
+        // the magic "do not update" value
+        // We don't do this for new runs because then they would never complete
+        // quality should be used to drop bad components
+        psStringAppend(&query, " AND magicDSFile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT);
+    }
 
     if (!p_psDBRunQuery(config->dbh, query)) {
@@ -1744 +1755 @@
     }
 
+    // we do not update components with the magic fault value. They are non-updateable
+    // (But can be recovered with "magicdstool -revertdestreakedfile -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
+    psStringAppend(&query, " AND (magicDSFile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
+
     if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");

branches/eam_branches/ipp-20101205/ippTools/src/magictool.c

@@ -627 +627 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0");

branches/eam_branches/ipp-20101205/ippTools/src/magictoolConfig.c

@@ -114 +114 @@
     psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0);
     psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
-    psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
 
     // -inputs

branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c

@@ -571 +571 @@
     PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
+    PXOPT_COPY_STR(config->args, where, "-reqType",     "reqType", "==");
+    PXOPT_COPY_STR(config->args, where, "-name",     "name", "LIKE");
     PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">=");
     PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<=");
@@ -586 +588 @@
         psStringAppend(&query, ", state = '%s'", state);
         if (!strcmp(state, "goto_cleaned")) {
-            psStringAppend(&stateCheck, " AND state != 'cleaned'");
+            psStringAppend(&stateCheck, " AND (state != 'cleaned' AND state != 'goto_cleaned')");
         }
     }

branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c

@@ -104 +104 @@
     psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0,        "search by fault code", 0);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0,      "search by reqType", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,      "search by reqType (LIKE comparsion)", NULL);
     psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
     psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL);

branches/eam_branches/ipp-20101205/ippTools/src/pxtools.h

@@ -51 +51 @@
 # define MAX_ROWS 10e9
 # define PXTOOL_MODE_NONE 0x0
+// we do not revert or update components with this fault value
+// This is the same value as PSTAMP_GONE
+# define PXTOOL_DO_NOT_REVERT_FAULT 26
 
 bool pxIsValidState(const char *state);

branches/eam_branches/ipp-20101205/ippTools/src/warptool.c

@@ -1485 +1485 @@
     PXOPT_COPY_S16(config->args, where, "-fault",      "warpSkyfile.fault", "==");
 
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
     if (!psListLength(where->list)
         && !psMetadataLookupBool(NULL, config->args, "-all")) {
@@ -1510 +1512 @@
     }
     psFree(where);
+
+    if (!fault) {
+        // If fault has not been supplied, don't revert update faults with the magic value
+        // We don't do this for new runs because then they would never complete
+        // quality should be used to drop bad components
+        psStringAppend(&query_updated, " AND warpSkyfile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT);
+    }
 
     if (!p_psDBRunQuery(config->dbh, query)) {
@@ -2003 +2012 @@
 // shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile
 
-static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state)
+static bool change_skyfile_data_state(pxConfig *config, psString data_state)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -2065 +2074 @@
 static bool tocleanedskyfileMode(pxConfig *config)
 {
-    return change_skyfile_data_state(config, "cleaned", "goto_cleaned");
+    return change_skyfile_data_state(config, "cleaned");
 }
 static bool tofullskyfileMode(pxConfig *config)
 {
-    return change_skyfile_data_state(config, "full", "update");
+    return change_skyfile_data_state(config, "full");
 }
 static bool topurgedskyfileMode(pxConfig *config)
 {
-    return change_skyfile_data_state(config, "purged", "goto_purged");
+    return change_skyfile_data_state(config, "purged");
 }
 static bool toscrubbedskyfileMode(pxConfig *config)
 {
-     return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
+     return change_skyfile_data_state(config, "scrubbed");
 }
 
@@ -2101 +2110 @@
     else {
       if (strcmp(state,"error_cleaned") == 0) {
-        change_skyfile_data_state(config,"error_cleaned","goto_cleaned");
+        change_skyfile_data_state(config,"error_cleaned");
       }
       else if (strcmp(state, "error_scrubbed") == 0) {
-        change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed");
+        change_skyfile_data_state(config,"error_scrubbed");
       }
       else if (strcmp(state, "error_purged") == 0) {
-        change_skyfile_data_state(config,"error_purged","goto_purged");
+        change_skyfile_data_state(config,"error_purged");
       }
       else {
@@ -2526 +2535 @@
         psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id);
     }
+    // we do not update components with the magic fault value. They are non-updateable
+    // (But can be recovered with "warptool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
+    psStringAppend(&query, " AND (warpSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
 
     if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) {