Changeset 20954 for branches/bills_081204/ippTools/src/difftool.c

Timestamp:

Dec 10, 2008, 6:11:39 PM (18 years ago)

Author:

bills

Message:

diff re-work nearly complete

File:

• branches/bills_081204/ippTools/src/difftool.c (modified) (12 diffs)

branches/bills_081204/ippTools/src/difftool.c

@@ -121 +121 @@
             0,          // ID
             "reg",      // state
+            workdir,
             label,
             reduction,
-            workdir,
             NULL,       // dvodb
             registered,
-            exp_id,
-            tess_id
+            tess_id,
+            exp_id
     );
     if (!run) {
@@ -234 +234 @@
     if (!diffInputSkyfileInsert(config->dbh,
             diff_id,
+            skycell_id,
             warp1 ? warp1 : PS_MAX_S64, // defined or NULL
+            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
             warp2 ? warp2 : PS_MAX_S64, // defined or NULL
-            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
             stack2 ? stack2 : PS_MAX_S64, // defined or NULL
-            skycell_id,
             tess_id
         )) {
@@ -304 +304 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diff_id", "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
     PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");
 
@@ -531 +531 @@
                            hostname,
                            good_frac,
-                           code
+                           code,
+                           0,       // diff_image_id
+                           0        // magic_ds_id
           )) {
         if (!psDBRollback(config->dbh)) {
@@ -795 +797 @@
             0,          // ID
             "reg",      // state
+            workdir,
             label,
             reduction,
-            workdir,
             NULL,       // dvodb
             registered,
-            exp_id,
-            tess_id
+            tess_id,
+            exp_id
     );
 
@@ -826 +828 @@
     if (!diffInputSkyfileInsert(config->dbh,
             run->diff_id,
+            skycell_id,
             input_warp_id,
+            input_stack_id,
             template_warp_id,
-            input_stack_id,
             template_stack_id,
-            skycell_id,
             tess_id
         )) {
@@ -936 +938 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    psMetadata *expWhere = psMetadataAlloc();
     psMetadata *warpWhere = psMetadataAlloc();
     psMetadata *stackWhere = psMetadataAlloc();
 
+    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
     PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "==");
@@ -953 +957 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
+    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery.sql");
+    psString query = pxDataGet("difftool_definebyquery_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -961 +967 @@
     }
 
-    if (newTemplates) {
-        // Warps that haven't been diffed OR warps that can take a newer template in the diff
-        psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)");
-    } else {
-        // Only warps that haven't been diffed
-        psStringAppend(&query, " WHERE diff_id IS NULL");
-    }
-
     psString warpQuery = NULL;
     psString stackQuery = NULL;
-
+    psString expQuery = NULL;
+
+    if (psListLength(expWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
+        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        expQuery = psStringCopy("\n");
+    }
+    psFree(expWhere);
     if (psListLength(warpWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
@@ -977 +984 @@
         psFree(whereClause);
     } else {
-        warpQuery = psStringCopy("");
+        warpQuery = psStringCopy("\n");
     }
     psFree(warpWhere);
+
+    if (!available) {
+        // diff what's available, even if warp run has some faults and is incomplete
+        psStringAppend(&warpQuery, " AND warpRun.state = 'full'");
+    }
+
+    // don't queue for exposures that have already been diff'd unless requested
+    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
+    }
 
     if (psListLength(stackWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
-        psStringAppend(&stackQuery, "\n AND %s", whereClause);
+        psStringAppend(&stackQuery, "\nAND %s", whereClause);
         psFree(whereClause);
     } else {
@@ -990 +1008 @@
     psFree(stackWhere);
 
-    psTrace("difftool", 1, query, warpQuery, stackQuery);
-
-    if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) {
+    psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);
+
+
+    if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -1019 +1038 @@
     }
 
+    // create temporary table
+    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    query = NULL;
+
+    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     long numGood = 0;                   // Number of good rows added
+    psS64 last_exp_id = 0;
     for (long i = 0; i < output->n; i++) {
         psMetadata *row = output->data[i]; // Output row from query
-
-        // Selected parameters
         bool mdok;                      // Status of MD lookup
-        const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
-        if (!mdok || !skycell_id) {
-            psWarning("skycell_id not found --- ignoring row %ld", i);
+
+        // Take the first warp for each exposure.
+        // The list is sorted by exposure id and warp_id and the warps are in descending
+        // order. 
+        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+            return false;
+        }
+        if (exp_id == last_exp_id) {
             continue;
         }
-        const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
-        if (!mdok || !tess_id) {
-            psWarning("tess_id not found --- ignoring row %ld", i);
-            continue;
-        }
-        psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id");
-        if (!mdok) {
-            psWarning("stack_id not found --- ignoring row %ld", i);
-            continue;
+        last_exp_id = exp_id;
+
+        // clear temporary table
+        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
         }
         psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
         if (!mdok) {
-            psWarning("warp_id not found --- ignoring row %ld", i);
+            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, warpQuery, stackQuery)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            return false;
+        }
+        psS64 num = psDBAffectedRows(config->dbh);
+
+        if (num == 0) {
+            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id);
             continue;
         }
-        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
-        if (!mdok) {
-            psWarning("exp_id not found --- ignoring row %ld", i);
+
+        if (!available && (num != skycell_count)) {
+            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" 
+                PRId64 " need %" PRId64, num, skycell_count, warp_id);
             continue;
         }
 
-        if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id,
-                          PS_MAX_S64, PS_MAX_S64, stack_id, exp_id, config)) {
-            psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
-                      warp_id, stack_id);
-            psErrorClear();
-            continue;
-        }
-
+        // ok we've got one create the diffRun
+        diffRunRow *run = diffRunRowAlloc(
+                0,          // ID
+                "reg",      // state
+                workdir,
+                label,
+                reduction,
+                NULL,       // dvodb
+                registered,
+                tess_id,
+                exp_id
+        );
+
+        if (!diffRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            return true;
+        }
+        run->diff_id = psDBLastInsertID(config->dbh);
+
+        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+        psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff");
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+
+        if (!setdiffRunState(config, run->diff_id, "new")) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                run->diff_id);
+            psFree(warpQuery);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            return false;
+        }
+
+        psArrayAdd(list, list->n, run);
         numGood++;
     }