Changeset 33415 for branches/meh_branches/ppstack_test/ippTools/src/addtool.c

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (2 props)
• ippTools/src/addtool.c (modified) (35 diffs)

branches/meh_branches/ppstack_test/ippTools/src

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool

branches/meh_branches/ppstack_test/ippTools/src/addtool.c

@@ -143 +143 @@
     pxAddLabelSearchArgs (config, where, "-filter",     "stackRun.filter", "=="); // define using camRun label
     }
-    
+        if (strcmp(stage, "staticsky_multi")== 0) {
+
+    pxAddLabelSearchArgs (config, where, "-label",     "staticskyRun.label", "=="); // define using camRun label
+    pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label
+    PXOPT_COPY_STR(config->args, where,  "-reduction", "staticskyyRun.reduction", "==");
+    //no filter here
+    }
     if (!psListLength(where->list)) {
         psFree(where);
@@ -237 +243 @@
     }
 
-
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      if (dvodb ) {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql");
+        // user supplied dvodb
+        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
+    } else {
+      psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi.sql");
+        // inherit dvodb from camRun, avoid matching NULL
+        psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)");
+    }
+    }
 
     if (!bare_query) {
@@ -272 +292 @@
     } else {
       //This picks only the unmagicked/uncensored ones
-      if (strcmp(stage,"cam") == 0) {
-        psStringAppend(&query, " AND (camRun.magicked = 0)");
-      }
+      //if (strcmp(stage,"cam") == 0) {
+        //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored.
+        //psStringAppend(&query, " AND (camRun.magicked = 0)");
+        //}
       if (strcmp(stage,"stack") == 0) {
         psStringAppend(&query, " AND (stackRun.magicked = 0)");
@@ -280 +301 @@
 
     }
+
+    //if we grab a group of camRun/stackRuns that have multiple camRun/stackRun pointing to the same exp_id/stack_id, and that exp_id/stack_id has never before been added to addRun, we need to group by exp_id/stack_id to ensure that only 1 of the camRun/stackRun for that exp_id is added to the addRun stage.
+
+    if (strcmp(stage,"cam") == 0) {
+        psStringAppend(&query, " GROUP BY exp_id"); 
+   }
+      if (strcmp(stage,"stack") == 0) {
+        psStringAppend(&query, " GROUP BY stack_id");
+      }
+    if (strcmp(stage,"staticsky") == 0) {
+        psStringAppend(&query, " GROUP BY stack_id");
+      }
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      psStringAppend(&query, " GROUP BY sky_id");  //some reason it needs this
+      }
+
 
     psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query);
@@ -391 +428 @@
     }
     }
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        if (num_inputs < 0) {
+          psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs");
+          return false;
+        }
+
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        if (!dvodb) {  //there's no staticsky.dvodb
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+        if (!workdir && !row->workdir) {
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+
+        psFree(row);
+    }
+    }
+
+
+
 
     // start a transaction so we don't end up with an exp without any associted
@@ -424 +500 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -465 +542 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -506 +584 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -530 +609 @@
       }
 
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psS64 stage_id =0; 
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        stage_id = row->sky_id;
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        for (int filter_id = 0; filter_id < num_inputs; filter_id++) {  
+        if (!pxaddQueueByCamID(config,
+                               stage,
+                               stage_id,
+                               filter_id,
+                               workdir     ? workdir   : row->workdir,
+                               reduction   ? reduction : row->reduction,
+                               label       ? label     : row->label,
+                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
+                               dvodb       ? dvodb     : NULL,
+                               note        ? note      : NULL,
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
+            psFree(row);
+            psFree(output);
+            return false;
+        }
+        }
+        psFree(row);
+    }
+      }
+
+
+
+
     psFree(output);
 
@@ -571 +703 @@
     }
     if (strcmp(stage, "staticsky")==0) {
+    query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
+    }
+    if (strcmp(stage, "staticsky_multi")==0) {
     query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
     }
@@ -612 +747 @@
     query = pxDataGet("addtool_find_pendingexp_staticsky.sql");
     }
-
+    if (strcmp(stage, "staticsky_multi")==0) { 
+    query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql");
+    }
+    
 
     if (!query) {
@@ -634 +772 @@
       psStringAppend(&query, " GROUP BY %s", "sky_id");
     }
-
+    if (strcmp(stage, "staticsky_multi") == 0) {
+      //this group by is needed to join against all the warps (to get camera)
+      psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1");
+    }
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -684 +825 @@
     PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false);
     PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
-
+    PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false);
     // generate restrictions
     psMetadata *where = psMetadataAlloc();
@@ -762 +903 @@
       }
     }
-
+    //if there is a stage_extra1, set it in addRun (it's not known until it is processed)
+    if (stage_extra1) {
+      psString setExtra = NULL;
+      psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1);
+      psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id);
+
+      if (!p_psDBRunQuery(config->dbh, setExtra)) {
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+
+        return false;
+      }
+    }
 
 
@@ -824 +979 @@
     query = pxDataGet("addtool_find_processedexp_staticsky.sql");
     }
+    if (strcmp (stage,"staticsky_multi") == 0) {
+    query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql");
+    }
+
    
     if (!query) {
@@ -905 +1064 @@
     PXOPT_COPY_S64(config->args, where, "-add_id",    "addRun.add_id",         "==");
     PXOPT_COPY_S64(config->args, where, "-stage_id",    "addRun.stage_id",         "==");
-    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
+    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", true, false);
     pxcamGetSearchArgs (config, where);
     pxAddLabelSearchArgs (config, where, "-label",    "addRun.label",     "==");
@@ -934 +1093 @@
          query = pxDataGet("addtool_revertprocessedexp_staticsky.sql");
       }
+   if (strcmp(stage, "staticsky_multi") == 0) {
+         query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql");
+      }
+
+
+     
       if (!query) {
             // rollback
@@ -1071 +1236 @@
 
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false);
-
-  //optional
+    //optional
   PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false);
   PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false);
@@ -1088 +1251 @@
   if (minidvodb_path) {
     minidvodbpath = minidvodb_path;
+  } else {
+    psError(PS_ERR_UNKNOWN, false, "require minidvodb_path");
+    return false;
   }
 
@@ -1095 +1261 @@
             minidvodb_group,
             minidvodbpath,
-            mergedvodb_path,
-            "new",
+            "new",
             0
             )) {
@@ -1118 +1283 @@
     psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id);
   }
-
-  if (!minidvodb_path) {
-    psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name);
-  }
-
   if (minidvodb_path) {
     psStringAppend(&minidvodb_path,"/%s",minidvodb_name);
@@ -1154 +1314 @@
     PXOPT_COPY_STR(config->args, where, "-state",     "state", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "==");
-    PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_group",     "minidvodb_group", "==");
 
@@ -1160 +1319 @@
     PXOPT_LOOKUP_STR(minidvodb_path,  config->args, "-set_minidvodb_path", false, false);
     PXOPT_LOOKUP_STR(state,  config->args, "-set_state", false, false);
-    PXOPT_LOOKUP_STR(mergedvodb_path,  config->args, "-set_mergedvodb_path", false, false);
     PXOPT_LOOKUP_STR(minidvodb_group,  config->args, "-set_minidvodb_group", false, false);
 
@@ -1195 +1353 @@
     }
 
-    if (mergedvodb_path) {
-      if (cnt) {
-        psStringAppend(&query, "%s", comma);
-      }
-      psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path);
-      cnt++;
-    }
-
     if (minidvodb_group) {
       if (cnt) {
@@ -1246 +1396 @@
   }
 
-  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL");
+  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL");
 
   psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL);
@@ -1294 +1444 @@
 
   //now flip new -> active
-  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL ");
+  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL ");
   psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group);
   if (!p_psDBRunQuery(config->dbh, query2)) {
@@ -1471 +1621 @@
     // required
   PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false);
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false);
 
     // optional
-  PXOPT_LOOKUP_U64(merge_order,     config->args, "-merge_order", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false);
-  PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false);
   PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
   
@@ -1530 +1676 @@
   minidvodbProcessedRow *row = minidvodbProcessedRowAlloc(
                pendingRow->minidvodb_id,
-               merge_order,
                dtime_resort,
                dtime_relphot,
-               dtime_verify,
-               dtime_merge,
                dtime_script,
                epoch,
-               mergedvodb_path,
                fault
                );
@@ -1552 +1694 @@
   }
 
-
-
-
-  //this finds the # of merged things (for the merge order)
-  psString query3 = NULL;
-  psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group);
-
-  if (!p_psDBRunQuery(config->dbh, query3)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query3);
-    return false;
-  }
-  psArray *output2 = p_psDBFetchResult(config->dbh);
-  if (!output2) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    return false;
-  }
-  if (!psArrayLength(output2)) {
-    psTrace("addtool", PS_LOG_INFO, "no rows found");
-    psFree(output);
-    return true;
-  }
-  bool status;
-  psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)");
-  if (!status) {
-
-    psAbort("failed to lookup value for count column");
-    return false;
-  }
-  psString final = NULL;
-  psStringAppend(&final, "%" PRIu64, m_order);
-    //return false;
-  psFree(query3);
-  psFree(output2);
-
-
-
-  //update the merge_order
-
-  psString query4 = NULL;
-  psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order);
-  psStringAppend(&query4," where minidvodb_id = %" PRIu64,  minidvodb_id);
-  //printf("%s", query4);
-  if (!p_psDBRunQuery(config->dbh, query4)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query4);
-    return false;
-  }
-  //
-psFree(query4);
-
-
-
-// since there is only one exp per 'new' set mindvodbRun.state = 'merged'
-
   psString query2 = NULL ;
   psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id);
@@ -1627 +1706 @@
     return false;
   }
-
-
-
-
-
-
-
-
   psFree(row);
   psFree(pendingRow);
@@ -1647 +1718 @@
     return false;
   }
-
-
-  //print the merge_order (why not!)
-  printf("%s", final);
-  psFree(final);
-
   return true;
 }
@@ -1661 +1726 @@
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "==");
-  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "==");
-  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "==");
   PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
   PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
@@ -1811 +1876 @@
 
   PXOPT_LOOKUP_U64(minidvodb_id,  config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_U64(merge_order,  config->args, "-set_merge_order", false, false);
   PXOPT_LOOKUP_S16(fault,  config->args, "-set_fault", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot,  config->args, "-set_dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort,  config->args, "-set_dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge,  config->args, "-set_dtime_merge", false, false);
-PXOPT_LOOKUP_F32(dtime_verify,  config->args, "-set_dtime_verify", false, false);
 PXOPT_LOOKUP_F32(dtime_script,  config->args, "-set_dtime_script", false, false);
   PXOPT_COPY_S64(config->args, where, "-minidvodb_id",     "minidvodbProcessed.minidvodb_id", "==");
@@ -1836 +1898 @@
   }
 
-  if (merge_order) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-
-    psStringAppend(&query, " merge_order = %" PRId64, merge_order);
-    cnt++;
-  }
-
   if (dtime_relphot) {
     if (cnt) {
@@ -1861 +1914 @@
   }
 
-  if (dtime_merge) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_merge = %f", dtime_merge);
-    cnt++;
- }
-if (dtime_verify) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_verify = %f", dtime_verify);
-    cnt++;
- }
-if (dtime_script) {
+  if (dtime_script) {
     if (cnt) {
       psStringAppend(&query, "%s", comma);

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool
@@ -43 +44 @@
 minidvodbtool
 laptool
+vptool