Changeset 24244 for branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c

Timestamp:

May 26, 2009, 1:59:32 PM (17 years ago)

Author:

beaumont

Message:

merged with trunk

Location:

branches/cnb_branches/cnb_branch_20090301

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/difftool.c (modified) (37 diffs)

branches/cnb_branches/cnb_branch_20090301/ippTools/src

@@ -33 +33 @@
 pstamptool
 disttool
+receivetool

@@ -33 +33 @@
 pstamptool
 disttool
+receivetool

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c

@@ -40 +40 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definebyqueryMode(pxConfig *config);
+static bool definewarpstackMode(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -79 +79 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -116 +116 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -133 +134 @@
             registered,
             tess_id,
-            exp_id,
+            bothways,
+            exposure,
             false
     );
     if (!run) {
         psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
-        return true;
+        return false;
     }
     if (!diffRunInsertObject(config->dbh, run)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(run);
-        return true;
+        return false;
     }
 
@@ -150 +152 @@
 
     if (!diffRunPrintObject(stdout, run, !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print object");
-            psFree(run);
-            return false;
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(run);
+        return false;
     }
 
@@ -412 +414 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
-    PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -485 +487 @@
     PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
-    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
     // optional
@@ -506 +508 @@
     PXOPT_LOOKUP_F32(kernel_xy, config->args, "-kernel_xy", false, false);
     PXOPT_LOOKUP_F32(kernel_yy, config->args, "-kernel_yy", false, false);
+    PXOPT_LOOKUP_F32(deconv_max, config->args, "-deconv_max", false, false);
     PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
@@ -519 +522 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
+                           "full",
                            bg,
                            bg_stdev,
@@ -533 +536 @@
                            kernel_xy,
                            kernel_yy,
+                           deconv_max,
                            sources,
                            dtime_diff,
@@ -540 +544 @@
                            hostname,
                            good_frac,
-                           code,
+                           fault,
+                           quality,
                            magicked
           )) {
@@ -578 +583 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "diffSkyfile.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
@@ -655 +660 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
     PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",     "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");
 
     if (!psDBTransaction(config->dbh)) {
@@ -690 +695 @@
         psFree(query);
 
-        psLogMsg("diftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
+        psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
     }
 
@@ -766 +771 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
-                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -797 +801 @@
             registered,
             tess_id,
-            exp_id,
+            false,
+            false,
             false       // magicked
     );
@@ -877 +882 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -897 +901 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
-        return false;
-    }
-    if (input_warp_id && !exp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -912 +911 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
-                      exp_id ? exp_id : PS_MAX_S64,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -930 +928 @@
 
 
-static bool definebyqueryMode(pxConfig *config)
+static bool definewarpstackMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -955 +953 @@
     PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery_part1.sql");
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1052 +1051 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
     // create temporary table
-    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1065 +1075 @@
     query = NULL;
 
-    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
@@ -1178 +1188 @@
                 registered,
                 tess_id,
-                exp_id,
+                false,                  // bothways
+                true,                   // exposure
                 false       // magicked
         );
@@ -1269 +1280 @@
     PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "templateRawExp.obs_mode", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-input_label", "inputWarpRun.label", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-template_label", "templateWarpRun.label", "==");
+    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
     PXOPT_COPY_F32(config->args, selectWhere, "-timediff",
                    "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");
-    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
+
+    // Haversine formula for great circle distance
+    PXOPT_COPY_F32(config->args, selectWhere, "-distance",
+                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
+                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
+                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
+
+    PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);
+
+    psMetadataAddF32(selectWhere, PS_LIST_TAIL,
+                     "TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs))",
+                     PS_META_DUPLICATE_OK, backwards ? "<" : ">", 0.0);
 
     // Restrictions for inserting skycells
@@ -1289 +1314 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        return false;
+    }
+
+    if (!rerun) {
+        // Need to build table of exposures with diffs
+        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
+        if (!tempCreate) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
+            psFree(tempCreate);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(tempCreate);
+
+        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
+        if (!tempInsert) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString where = psStringCopy(""); // WHERE for insertion
+        if (label) {
+            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
+            psFree(tempInsert);
+            psFree(where);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(where);
+        psFree(tempInsert);
+    }
+
+    // Get list of warps to diff
     psString select = pxDataGet("difftool_definewarpwarp_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1306 +1400 @@
     psFree(selectWhere);
 
+    if (!available) {
+        psStringAppend(&whereClause,
+                       "\n%s inputWarpRun.state = 'full'"
+                       "AND templateWarpRun.state = 'full'",
+                       whereClause ? "AND" : "WHERE");
+    }
+
     if (!rerun) {
-        psStringAppend(&whereClause, "\n%s diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
-    }
-
-    if (!available) {
-        psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
-    }
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(select);
-        psFree(whereClause);
-        psFree(insertWhere);
-        return false;
+        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
     }
 
     if (!p_psDBRunQueryF(config->dbh, select,
-                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                          whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
@@ -1368 +1457 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, results, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(results);
+            return false;
+        }
+        psFree(results);
+        return true;
+    }
+
     psString insert = pxDataGet("difftool_definewarpwarp_insert.sql"); // Insertion for each new run
 
     if (psListLength(insertWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(insertWhere, NULL);
-        psStringAppend(&insert, "\n%s", whereClause);
+        psStringAppend(&insert, "\nAND %s", whereClause);
         psFree(whereClause);
     }
@@ -1382 +1482 @@
         psMetadata *row = results->data[i]; // Result row from query
 
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
         psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
-        psS64 template_id = psMetadataLookupS64(NULL, row, "template_warp_id");
+        const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
-        if (!exp_id || !input_id || !template_id || !tess_id) {
+        if (!input_id || !template || !tess_id) {
             psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
             psFree(list);
@@ -1398 +1497 @@
 
         diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
-                                          tess_id, exp_id, false); // Run to insert
+                                          tess_id, true, true, false); // Run to insert
         if (!diffRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1413 +1512 @@
 
         // Convert identifiers to string, for insertion into query
-        psString diff = NULL, input = NULL, template = NULL; // String versions of identifiers
+        psString diff = NULL, input = NULL; // String versions of identifiers
         psStringAppend(&diff, "%" PRId64, run->diff_id);
         psStringAppend(&input, "%" PRId64, input_id);
-        psStringAppend(&template, "%" PRId64, template_id);
 
         if (!p_psDBRunQueryF(config->dbh, insert, diff, input, template, input, template)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(input);
-            psFree(template);
             psFree(diff);
             psFree(run);
@@ -1433 +1530 @@
         }
         psFree(input);
-        psFree(template);
         psFree(diff);
 
@@ -1482 +1578 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanuprun.sql");
@@ -1547 +1643 @@
         PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
@@ -1662 +1758 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
 
-    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, code)) {
+    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree (where);
@@ -1737 +1833 @@
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
 
   FILE *f = fopen (outfile, "w");
@@ -1782 +1879 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
       return false;
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
       psFree(output);
-      return true;
+      return false;
+    }
+
+    if (clean) {
+        bool success = true;
+        if (!strcmp(tables[i].tableName, "diffRun")) {
+            success = pxSetStateCleaned("diffRun", "state", output);
+#ifdef notyet
+        // diffSkyfile doesn't have dataState yet
+        } else if (!strcmp(tables[i].tableName, "diffSkyfile")) {
+            success = pxSetStateCleaned("diffSkyfile", "data_state", output);
+#endif
+        }
+        if (!success) {
+            psFree(output);
+            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+            return false;
+        }
     }
 

@@ -33 +33 @@
 pstamptool
 disttool
+receivetool