Changeset 18923 for branches/eam_branch_20080719/ippTools/src/magictool.c

Timestamp:

Aug 5, 2008, 2:07:07 PM (18 years ago)

Author:

eugene

Message:

updates from HEAD

File:

• branches/eam_branch_20080719/ippTools/src/magictool.c (modified) (9 diffs)

branches/eam_branch_20080719/ippTools/src/magictool.c

@@ -110 +110 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    // Required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+
+    // Optional
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
-
-    // create warped skycells temp table
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // Create temporary table of the best diffs
     {
-        psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql");
+        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -127 +131 @@
             return false;
         }
-    }
-
-    // find warped skycells
+        psFree(query);
+    }
+
+    // Insert list of best diffs into temporary table
     {
-        psString query = pxDataGet("magictool_find_complete_warpruns.sql");
+        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             return false;
         }
+
+        psMetadata *where = psMetadataAlloc();
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+        PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&query, " AND %s", whereClause);
+            psFree(whereClause);
+        }
+        psFree(where);
+
+        psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql");
+        if (!groupby) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(query);
+            return false;
+        }
+
+        psStringAppend(&query, " %s", groupby);
+        psFree(groupby);
 
         if (!p_psDBRunQuery(config->dbh, query)) {
@@ -145 +171 @@
     }
 
-    // find the diff_id's of the warped skycells
+    // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql");
+        psString query = pxDataGet("magictool_definebyquery_select_part1.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             return false;
         }
+
+        {
+            psMetadata *where = psMetadataAlloc();
+            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+            PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+
+            if (psListLength(where->list)) {
+                psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringAppend(&query, " AND %s", whereClause);
+                psFree(whereClause);
+            }
+            psFree(where);
+        }
+
+        psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql");
+        if (!part2) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        psStringAppend(&query, " %s", part2);
+        psFree(part2);
 
         if (!p_psDBRunQuery(config->dbh, query)) {
@@ -180 +228 @@
     }
 
-    // pre-allocate enough hash buckets for 1 unique warp_id per row
-    psHash *groups = psHashAlloc(psArrayLength(output));
-    // iterate over result set, and group by warp_id
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query
+
+    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     for (long i = 0; i < psArrayLength(output); i++) {
-        // lookup the key we're grouping by
-        bool status = false;
-        psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id");
-        if (!status) {
-            psAbort("failed to lookup value for warp_id");
-        }
-
-        psString key = psDBIntToString(warp_id);
-        // look to see if there is a bin for this key
-        psArray *groupBin = psHashLookup(groups, key);
-        if (!groupBin) {
-            // if not, create one
-            groupBin = psArrayAlloc(0);
-            psHashAdd(groups, key, groupBin);
-        }
-        psFree(key);
-
-        // add this row to the bin
-        psArrayAdd(groupBin, 0, output->data[i]);
-        psFree(groupBin);
-    }
-    psFree(output);
-
-    // create a magic run
-    psArray *grouped =  psHashToArray(groups);
-    psFree(groups);
-
-    // iterate over array of grouped warp_ids
-    for (long i = 0; i < psArrayLength(grouped); i++) {
-        psArray *group = grouped->data[i];
+        psMetadata *row = output->data[i]; // Row of interest
+        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(
-            0,                          // ID
-            "reg",                      // state
-            workdir,                    // workdir
-            "dirty",                    // workdir_state
-            label,                      // label
-            dvodb,                      // dvodb
-            registered,                 // registered
-            0                           // fault
-            );
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
         }
+
         if (!magicRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(run);
-            psFree(grouped);
-            return false;
-        }
-
-        // get the assigned warp_id
-        psS64 magic_id = psDBLastInsertID(config->dbh);
-
-        // insert all rows in this bin into the new magicRun
-        for (long j = 0; j < psArrayLength(group); j++) {
-            psMetadata *row = group->data[j];
-
-            bool status = false;
-            psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id");
-            if (!status) {
-                psAbort("failed to lookup value for diff_id");
+            psFree(insert);
+            psFree(output);
+            psFree(list);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
             }
-            psString node = psMetadataLookupStr(&status, row, "skycell_id");
-            if (!status) {
-                psAbort("failed to lookup value for diff_id");
+            return false;
+        }
+
+        psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier
+        run->magic_id = magic_id;
+
+        psArrayAdd(list, list->n, run);
+        psFree(run);
+
+        // Create a suitable insertion query for this run
+        psString thisInsert = psStringCopy(insert);
+        {
+            psString idString = NULL;
+            psStringAppend(&idString, "%" PRId64, magic_id);
+            psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@");
+            psFree(idString);
+        }
+        {
+            psString idString = NULL;
+            psStringAppend(&idString, "%" PRId64, exp_id);
+            psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
+            psFree(idString);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id, exp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(thisInsert);
+            psFree(insert);
+            psFree(output);
+            psFree(list);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
             }
-
-            if (!magicInputSkyfileInsert(config->dbh,
-                                         magic_id,
-                                         diff_id,
-                                         node)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-                psFree(grouped);
-                return false;
-            }
-        }
-
-        // set magicRun.state to 'run'
-        return setmagicRunState(config, magic_id, "run");
-    }
-    psFree(grouped);
+            return false;
+        }
+        psFree(thisInsert);
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(output);
+
+    if (!magicRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+
+    psFree(list);
 
     return true;
@@ -277 +316 @@
     // required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
 
     // optional
@@ -286 +326 @@
     magicRunRow *run = magicRunRowAlloc(
             0,          // ID
+            exp_id,
             "reg",      // state
             workdir,
@@ -658 +699 @@
     }
 
+    if (!node->data) {
+        // It's a leaf node, not done
+        return false;
+    }
+
     if (psMetadataLookupBool(NULL, node->data, "done")) {
         // It's already done
@@ -671 +717 @@
         while ((child = psListGetAndIncrement(iter))) {
             psMetadata *data = child->data;
+            if (!data) {
+                // Child is a leaf node, not done
+                psFree(iter);
+                psFree(work);
+                return false;
+            }
 
             bool status = false;
@@ -812 +864 @@
 
     // find the root of the tree
-    pxNode *root = psHashLookup(forest, "root");
+    pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
     psFree(forest);
-    // pxTreePrint(stdout, root);
+    //    pxTreePrint(stdout, root);
 
     // crawl through the tree and looking for nodes with children that are all