Changeset 35149

Timestamp:

Feb 11, 2013, 4:50:12 PM (13 years ago)

Author:

watersc1

Message:

Changes that should not conflict with the trunk. This is the camera stage background continuity and the pattern.median example code.

Location:

branches/czw_branch/20130211

Files:

• dbconfig (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 prop)
• ippTools (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (3 diffs)
• ippTools/src (modified) (1 prop)
• ippTools/src/difftool.c (modified) (10 diffs)
• ippconfig/gpc1/ppImage.config (modified) (4 diffs)
• ippconfig/recipes/ppImage.config (modified) (1 diff)
• ppImage/src/ppImage.h (modified) (1 diff)
• ppImage/src/ppImageDetrendPattern.c (modified) (2 diffs)
• ppImage/src/ppImageLoop.c (modified) (1 diff)
• ppImage/src/ppImageOptions.c (modified) (2 diffs)
• psModules/src/detrend/pmPattern.c (modified) (3 diffs)
• psModules/src/detrend/pmPattern.h (modified) (1 diff)

branches/czw_branch/20130211/ippTools/share/pxadmin_create_tables.sql

@@ -1844 +1844 @@
     data_group VARCHAR(64),           -- group for data
     dist_group VARCHAR(64),           -- group for distribution
-    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    reduction VARCHAR(64),            -- reduction class (for altering recipe)
+    alt_cam_label VARCHAR(64),        -- label for alternate version of astrometry (optional)
     note VARCHAR(255),        -- note
     registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
@@ -2162 +2163 @@
     surveyID    INT(8),
     releaseName VARCHAR(64),
-    state       VARCHAR(16),    -- active, pending archive drop 
+    relase_state VARCHAR(16),   -- active, pending, archive, drop 
     dataRelease INT(8),         -- PSPS dataRelease
+    priority    INT(8),
     PRIMARY KEY(rel_id),
     KEY(releaseName),
     KEY(state),
+    KEY(priority),
     FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
 ) ENGINE=Innodb DEFAULT CHARSET=latin1;
@@ -2192 +2195 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
-
--- This comment line is here to avoid empty query error.
+CREATE TABLE relStack (
+    relstack_id BIGINT AUTO_INCREMENT,
+    rel_id      INT(8),
+    skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
+                                -- for this release. We get back to stack from here
+    skycell_id  varchar(64),
+    tess_id     varchar(64),
+    state       VARCHAR(16),    -- released, pending, archive, drop  
+    flags       INT UNSIGNED,
+    stack_type  VARCHAR(16),    -- nightly, deep, reference
+    zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
+    zpt_stdev   FLOAT,
+    fwhm_major  FLOAT,          -- seeing as measured by staticsky
+    path_base   VARCHAR(255),   -- path_base of any supporting files,
+    registered  DATETIME,
+    time_stamp  DATETIME,
+    PRIMARY KEY (relstack_id),
+    KEY (tess_id, skycell_id),
+    KEY (rel_id, tess_id, skycell_id),
+    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
+    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+
+-- These comment lines are here to avoid an empty query error.
 -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.

branches/czw_branch/20130211/ippTools/src/difftool.c

@@ -41 +41 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definewarpstackMode(pxConfig *config);
+//static bool definewarpstackMode(pxConfig *config);
+static bool definewarpstackModeNew(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool definestackstackMode(pxConfig *config);
@@ -94 +95 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackModeNew);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK,      definestackstackMode);
@@ -1226 +1227 @@
 }
 
-
-static bool definewarpstackMode(pxConfig *config)
+static bool definewarpstackModeNew(pxConfig *config)
 {
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
+  PS_ASSERT_PTR_NON_NULL(config,false);
     psMetadata *expWhere = psMetadataAlloc();
     psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp
-    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
+    //    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
     psMetadata *stackWhere = psMetadataAlloc();
 
@@ -1243 +1242 @@
     PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "==");
     PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "==");
-    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
-    PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
+    //    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_F32(config->args, warp1Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
@@ -1268 +1267 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
-    // find all things to queue
-    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    psString warp1Query = NULL;
-    psString warp2Query = NULL;
-    psString stackQuery = NULL;
-    psString expQuery = NULL;
-
+    psString newQuery = NULL;
+    psStringAppend(&newQuery, "select exp_id,warp_id,rawExp.filter,warpRun.label,warpRun.data_group,warpRun.tess_id,warpRun.data_group,");
+    psStringAppend(&newQuery, "warpSkyfile.skycell_id,stack_id,stackRun.skycell_id,stackRun.tess_id,stackRun.label,stackRun.data_group,diff_id ");
+    psStringAppend(&newQuery, "from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ");
+    psStringAppend(&newQuery, "JOIN warpSkyfile USING(warp_id) JOIN stackRun USING(skycell_id,filter) ");
+    psStringAppend(&newQuery, "LEFT JOIN diffInputSkyfile ON (warp_id = diffInputSkyfile.warp1 AND stack_id = diffInputSkyfile.stack2 AND diffInputSkyfile.skycell_id = stackRun.skycell_id) WHERE 1");
+    
     if (psListLength(expWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
-        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psStringAppend(&newQuery, "\n AND %s", whereClause);
         psFree(whereClause);
     } else {
-        expQuery = psStringCopy("\n");
+      //        newQuery = psStringCopy("\n");
     }
     psFree(expWhere);
     if (psListLength(warp1Where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL);
-        psStringAppend(&warp1Query, "\n AND %s", whereClause);
+        psStringAppend(&newQuery, "\n AND %s", whereClause);
         psFree(whereClause);
     } else {
-        warp1Query = psStringCopy("\n");
+      //        waQuery = psStringCopy("\n");
     }
     psFree(warp1Where);
-    if (psListLength(warp2Where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL);
-        psStringAppend(&warp2Query, "\n AND %s", whereClause);
+    
+    // don't queue for exposures that have already been diff'd unless requested
+    //    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&newQuery, "\nAND diff_id IS NULL");
+    } else {
+      //        diffQuery = psStringCopy("\n");
+    }
+
+    if (psListLength(stackWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
+        psStringAppend(&newQuery, "\nAND %s", whereClause);
         psFree(whereClause);
     } else {
-        warp2Query = psStringCopy("\n");
-    }
-    psFree(warp2Where);
-
-    // don't queue for exposures that have already been diff'd unless requested
-    psString diffQuery = NULL;
-    if (! (reRun || newTemplates) ) {
-        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
-    } else {
-        diffQuery = psStringCopy("\n");
-    }
-
-    if (psListLength(stackWhere->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
-        psStringAppend(&stackQuery, "\nAND %s", whereClause);
-        psFree(whereClause);
-    } else {
-        stackQuery = psStringCopy("");
+      //        stackQuery = psStringCopy("");
     }
     psFree(stackWhere);
 
-    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
-
+    //    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
+    fprintf(stderr,"%s",newQuery);
+    return(true);
+#if (0)    
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1329 +1317 @@
     }
 
+    
     if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1367 +1356 @@
         return true;
     }
-
-    if (pretend) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-        psFree(output);
-        return true;
-    }
-
-    // create temporary table
-    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        return false;
-    }
-    psFree(warp1Query);
-    psFree(query);
-    query = NULL;
-
-    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
-
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     long numGood = 0;                   // Number of good rows added
@@ -1401 +1362 @@
         psMetadata *row = output->data[i]; // Output row from query
         bool mdok;                      // Status of MD lookup
-
-        // Take the first warp for each exposure.
-        // The list is sorted by exposure id and warp_id and the warps are in descending
-        // order.
-        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        if (exp_id == last_exp_id) {
-            continue;
-        }
-        last_exp_id = exp_id;
-
-        // clear temporary table
-        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        psString filter = psMetadataLookupStr(&mdok, row, "filter");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "filter not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group");
-        if (!mdok) {
-          psError(PXTOOLS_ERR_PROG, false, "warp data_group not found");
-          psFree(warp2Query);
-          psFree(stackQuery);
-          psFree(skycell_query);
-          if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-          }
-          return false;
-        }
-        if (!data_group && warp_data_group) {
-          data_group = warp_data_group;
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 num = psDBAffectedRows(config->dbh);
-
-        if (num == 0) {
-            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id);
-            continue;
-        }
-
-        if (!available && (num != skycell_count)) {
-            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64
-                    " but need %" PRId64, num, warp_id, skycell_count);
-            continue;
-        }
-
         // ok we've got one create the diffRun
         diffRunRow *run = diffRunRowAlloc(
@@ -1537 +1386 @@
                 note
         );
-
-        if (!diffRunInsertObject(config->dbh, run)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(run);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        run->diff_id = psDBLastInsertID(config->dbh);
-
-        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            psFree(query);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-            return false;
-        }
-        psFree(query);
-        query = NULL;
         psStringAppend(&query,
                        "INSERT INTO diffInputSkyfile(diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id) SELECT diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id from skycellsToDiff");
@@ -1613 +1437 @@
 
     return true;
+#endif
 }
-
+#if (0)   
+static bool definewarpstackMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *expWhere = psMetadataAlloc();
+    psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp
+    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
+    psMetadata *stackWhere = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "==");
+    PXOPT_COPY_STR(config->args, expWhere, "-comment", "comment", "LIKE");
+    PXOPT_COPY_S64(config->args, warp1Where, "-warp_id", "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "==");
+    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
+
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
+    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    // find all things to queue
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString warp1Query = NULL;
+    psString warp2Query = NULL;
+    psString stackQuery = NULL;
+    psString expQuery = NULL;
+
+    if (psListLength(expWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
+        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        expQuery = psStringCopy("\n");
+    }
+    psFree(expWhere);
+    if (psListLength(warp1Where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL);
+        psStringAppend(&warp1Query, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        warp1Query = psStringCopy("\n");
+    }
+    psFree(warp1Where);
+    if (psListLength(warp2Where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL);
+        psStringAppend(&warp2Query, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        warp2Query = psStringCopy("\n");
+    }
+    psFree(warp2Where);
+
+    // don't queue for exposures that have already been diff'd unless requested
+    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
+    } else {
+        diffQuery = psStringCopy("\n");
+    }
+
+    if (psListLength(stackWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
+        psStringAppend(&stackQuery, "\nAND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        stackQuery = psStringCopy("");
+    }
+    psFree(stackWhere);
+
+    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+          case PS_ERR_DB_CLIENT:
+            psError(PXTOOLS_ERR_SYS, false, "database error");
+            break;
+          case PS_ERR_DB_SERVER:
+            psError(PXTOOLS_ERR_PROG, false, "database error");
+            break;
+          default:
+            psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            break;
+        }
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    // create temporary table
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(warp1Query);
+    psFree(query);
+    query = NULL;
+
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
+
+    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
+    long numGood = 0;                   // Number of good rows added
+    psS64 last_exp_id = 0;
+    for (long i = 0; i < output->n; i++) {
+        psMetadata *row = output->data[i]; // Output row from query
+        bool mdok;                      // Status of MD lookup
+
+        // Take the first warp for each exposure.
+        // The list is sorted by exposure id and warp_id and the warps are in descending
+        // order.
+        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        if (exp_id == last_exp_id) {
+            continue;
+        }
+        last_exp_id = exp_id;
+
+        // clear temporary table
+        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString filter = psMetadataLookupStr(&mdok, row, "filter");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "filter not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group");
+        if (!mdok) {
+          psError(PXTOOLS_ERR_PROG, false, "warp data_group not found");
+          psFree(warp2Query);
+          psFree(stackQuery);
+          psFree(skycell_query);
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          return false;
+        }
+        if (!data_group && warp_data_group) {
+          data_group = warp_data_group;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 num = psDBAffectedRows(config->dbh);
+
+        if (num == 0) {
+            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id);
+            continue;
+        }
+
+        if (!available && (num != skycell_count)) {
+            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64
+                    " but need %" PRId64, num, warp_id, skycell_count);
+            continue;
+        }
+
+        // ok we've got one create the diffRun
+        diffRunRow *run = diffRunRowAlloc(
+                0,            // ID
+                "reg",        // state
+                workdir,
+                label,
+                data_group ? data_group : label,
+                dist_group,
+                reduction,
+                NULL,         // dvodb
+                registered,
+                tess_id,
+                bothways,     // bothways (default is false)
+                true,         // exposure
+                0,            // magicked
+                NULL,         // software version
+                0,            // mask stat npix
+                NAN,          // static
+                NAN,          // dynamic
+                NAN,          // magic
+                NAN,          // advisory
+                IPP_DIFF_MODE_WARP_STACK,
+                note
+        );
+
+        if (!diffRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        run->diff_id = psDBLastInsertID(config->dbh);
+
+        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+        psStringAppend(&query,
+                       "INSERT INTO diffInputSkyfile(diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id) SELECT diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id from skycellsToDiff");
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+
+        if (!setdiffRunState(config, run->diff_id, "new", false)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                run->diff_id);
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psArrayAdd(list, list->n, run);
+        numGood++;
+    }
+    psFree(output);
+    psFree(warp2Query);
+    psFree(stackQuery);
+    psFree(skycell_query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(list);
+        return false;
+    }
+
+    if (numGood && !diffRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+    psFree(list);
+
+    return true;
+}
+#endif
 static bool definewarpwarpMode(pxConfig *config)
 {

branches/czw_branch/20130211/ippconfig/gpc1/ppImage.config

@@ -159 +159 @@
   CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
   OVERSCAN           BOOL    TRUE            # Overscan subtraction
-  NONLIN             BOOL    TRUE            # apply non-linearity correction 
+  NONLIN             BOOL    FALSE            # apply non-linearity correction 
   BIAS               BOOL    FALSE           # Bias subtraction
   DARK               BOOL    TRUE            # Dark subtraction
@@ -169 +169 @@
   MASK.BUILD         BOOL    TRUE            # Build internal mask?
   MASK.BURNTOOL      BOOL    TRUE            # Mask potential burntool trails
-  PATTERN.ROW        BOOL    TRUE            # Row pattern correction
+  PATTERN.ROW        BOOL    FALSE           # Row pattern correction
   PATTERN.CELL       BOOL    FALSE           # Cell pattern correction
   PATTERN.CONTINUITY BOOL    TRUE            # Cell pattern correction using edge continuity
+  PATTERN.MEDIAN     BOOL    TRUE
   FRINGE             BOOL    TRUE            # Fringe subtraction
   BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
@@ -1229 +1230 @@
   CHIP.VARIANCE.FITS BOOL    TRUE            # Save chip-mosaic-ed image? 
   OVERSCAN           BOOL    TRUE            # Overscan subtraction
-  NONLIN             BOOL    TRUE            # apply non-linearity correction 
+  NONLIN           BOOL    TRUE            # apply non-linearity correction 
   BIAS               BOOL    FALSE           # Bias subtraction
   DARK               BOOL    TRUE            # Dark subtraction
@@ -1238 +1239 @@
   MASK.BUILD         BOOL    TRUE            # Build internal mask?
   MASK.BURNTOOL      BOOL    TRUE            # Mask potential burntool trails
-  PATTERN.ROW        BOOL    FALSE           # Row pattern correction
-  PATTERN.CELL       BOOL    FALSE           # Cell pattern correction
+  PATTERN.ROW        BOOL    TRUE           # Row pattern correction
+  PATTERN.CELL       BOOL    TRUE            # Cell pattern correction
   FRINGE             BOOL    TRUE            # Fringe subtraction
   BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?

branches/czw_branch/20130211/ippconfig/recipes/ppImage.config

@@ -30 +30 @@
 PATTERN.CONTINUITY.WIDTH S32  50
 PATTERN.CONTINUITY.SUBSET BOOL TRUE
+PATTERN.MEDIAN     BOOL    FALSE
 FRINGE             BOOL    FALSE           # Fringe subtraction
 PHOTOM             BOOL    FALSE           # Source identification and photometry

branches/czw_branch/20130211/ppImage/src/ppImage.h

@@ -42 +42 @@
     bool doPatternContinuity;           // Cell continuity correction
     bool doBackgroundContinuity;        // Do mosaic continuity correction
+    bool doPatternMedian;               // Do median based pattern correction
     bool doFringe;                      // Fringe subtraction
     bool doPhotom;                      // Source identification and photometry

branches/czw_branch/20130211/ppImage/src/ppImageDetrendPattern.c

@@ -18 +18 @@
     pmCell *cell = NULL;
 
-    assert(options->doPatternRow || options->doPatternCell || options->doPatternContinuity); // do not call if not needed
+    assert(options->doPatternMedian || options->doPatternRow ||
+           options->doPatternCell || options->doPatternContinuity); // do not call if not needed
     assert(inputView->chip != -1);
     assert(inputView->cell == -1);
@@ -35 +36 @@
     // chip.
 
+    if (options->doPatternMedian) {
+      //      bool status;
+      int numCells = chip->cells->n;       // Number of cells
+      psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell?
+      //      pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT");
+      pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
+      
+
+      pmHDU *hdu = pmHDUFromChip(chip);
+      if (psMetadataLookupBool(NULL,hdu->header, "PTRN_MED")) {
+        psLogMsg("ppImage", PS_LOG_INFO, "Not performing median pattern correction as it has already been done.");
+        goto pattern_row;
+      }
+      for (int i = 0; i < chip->cells->n; i++) {
+        view->cell = i;
+        
+        pmCell *cell = chip->cells->data[i]; // Cell of interest
+        
+        if (cell->readouts->n > 1) {
+          psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction on video cell.");
+          continue;
+        }
+        
+        bool doPattern = false;
+        if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CONTINUITY.SUBSET")) {
+          ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
+        }
+        if (doPattern) {
+          tweak->data.U8[i] = 0xFF;
+        }
+      }
+      
+      // Tweak the cells
+      if (!pmPatternMedian(chip, tweak, options->patternCellBG, options->patternCellMean,
+                           options->maskValue, options->darkMask,true,true)) {
+        psFree(tweak);
+        psFree(view);
+        return false;
+      }
+      psFree(tweak);
+      psFree(view);
+      
+      psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_MED",PS_META_REPLACE,"PATTERN.MEDIAN correction applied",true);
+      
+    }
+ pattern_row:
     if (options->doPatternRow) {
         bool status;

branches/czw_branch/20130211/ppImage/src/ppImageLoop.c

@@ -174 +174 @@
 
         // Apply the pattern correction
-        if (options->doPatternRow || options->doPatternCell || options->doPatternContinuity) {
+        if (options->doPatternMedian || options->doPatternRow ||
+            options->doPatternCell || options->doPatternContinuity) {
           if (!ppImageDetrendPatternApply(config,chip,view,options)) {
             ESCAPE("Unable to apply pattern corrections");

branches/czw_branch/20130211/ppImage/src/ppImageOptions.c

@@ -36 +36 @@
     options->doPatternContinuity = false; // Cell continuity correction
     options->doBackgroundContinuity = false; // Chip level background continuity correction
+    options->doPatternMedian = false;   // Median based pattern correction
     options->doFringe        = false;   // Fringe subtraction
     options->doPhotom        = false;   // Source identification and photometry
@@ -259 +260 @@
     options->doPatternCell = psMetadataLookupBool(NULL, recipe, "PATTERN.CELL");
     options->doPatternContinuity = psMetadataLookupBool(NULL, recipe, "PATTERN.CONTINUITY");
+    options->doPatternMedian = psMetadataLookupBool(NULL, recipe, "PATTERN.MEDIAN");
 
     options->doMaskStats = psMetadataLookupBool(NULL, recipe, "MASK.STATS");

branches/czw_branch/20130211/psModules/src/detrend/pmPattern.c

@@ -4 +4 @@
 
 #include "pmFPA.h"
-
+#include "pmStack.h"
 #include "pmPattern.h"
 
@@ -1085 +1085 @@
         
       }
-      psTrace("psModules.detrend.cont",5, "OTA: %d (%d %d) (%d %d) A: %f B: %f C: %f D: %f",
+      psTrace("psModules.detrend.cont",5, "Continuity OTA: %d (%d %d) (%d %d) A: %f B: %f C: %f D: %f",
               i,x,y,xParity,yParity,
               A->data.F64[y][x],B->data.F64[y][x],C->data.F64[y][x],D->data.F64[y][x]);         
@@ -1302 +1302 @@
     return true;
 }
+
+
+bool pmPatternMedian(pmChip *chip, const psVector *tweak, psStatsOptions bgStat, psStatsOptions cellStat,
+                     psImageMaskType maskVal, psImageMaskType maskBad, bool doRows, bool doCols)
+{
+  //  fprintf(stderr,"Inside pmPatternMedian!\n");
+    PS_ASSERT_PTR_NON_NULL(chip, false);
+    PS_ASSERT_VECTOR_NON_NULL(tweak, false);
+    PS_ASSERT_VECTOR_SIZE(tweak, chip->cells->n, false);
+    PS_ASSERT_VECTOR_TYPE(tweak, PS_TYPE_U8, false);
+    
+    int numCells = tweak->n;            // Number of cells
+    numCells = chip->cells->n;
+    psStats *bgStats      = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics on background
+    psRandom *rng         = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
+    psVector *cellMedians = psVectorAlloc(numCells, PS_TYPE_F32); // Mask for means
+    psVector *cellSigmas  = psVectorAlloc(numCells, PS_TYPE_F32);
+    psVector *meanMask    = psVectorAlloc(numCells, PS_TYPE_VECTOR_MASK);
+    // Mask bits
+    enum {
+        PM_PATTERN_IGNORE = 0x01,       // Ignore this cell
+        PM_PATTERN_TWEAK  = 0x02,       // Tweak this cell
+        PM_PATTERN_ERROR  = 0x04,       // Error in calculating background
+        PM_PATTERN_ALL    = 0xFF,       // All causes
+    };
+
+    // Count number of cells to tweak
+    int numTweak = 0;                   // Number of cells to tweak
+    int numIgnore = 0;                  // Number of cells to ignore
+    psStatsInit(bgStats);
+
+    for (int i = 0; i < numCells; i++) { // Calculate cell level statistics, and set ignore mask
+      pmCell *cell = chip->cells->data[i]; // Cell of interest
+
+      if (!cell || !cell->data_exists || !cell->process ||
+          cell->readouts->n == 0 || cell->readouts->n > 1 || !cell->readouts->data[0]) {
+        numIgnore++;
+        meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PM_PATTERN_IGNORE;
+        cellMedians->data.F32[i] = NAN;
+        cellSigmas->data.F32[i] = NAN;
+        continue;
+      }
+      pmReadout *ro = cell->readouts->data[0];
+      if (tweak->data.U8[i]) {
+        meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PM_PATTERN_TWEAK;
+        numTweak++;
+      }
+      if (!psImageBackground(bgStats, NULL, ro->image, ro->mask, maskVal, rng)) {
+        psWarning("Unable to measure background for cell %d\n", i);
+        psErrorClear();
+        meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PM_PATTERN_ERROR;
+        continue;
+      }
+      cellMedians->data.F32[i] = bgStats->robustMedian;
+      cellSigmas->data.F32[i]  = bgStats->robustStdev;  
+    }
+    if (numTweak == 0) {
+      //        fprintf(stderr,"Inside pmPatternMedian but I'm claiming nothing to do!\n");
+        // Nothing to do
+        psFree(meanMask);
+        return true;
+    }
+
+    int num = 8;
+    
+    psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_QUARTILE);
+    //    psArray *inputs = psArrayAlloc(num);
+    psVector *inputs    = psVectorAlloc(num,PS_TYPE_S32);
+    psVector *pixelData = psVectorAlloc(inputs->n,PS_TYPE_F32);
+    psVector *pixelMask = psVectorAlloc(inputs->n,PS_TYPE_VECTOR_MASK);
+
+    for (int mode = 0; mode <= 1; mode++) {
+      if ((mode == 0)&&(!doRows)) {
+        continue;
+      }
+      if ((mode == 1)&&(!doCols)) {
+        continue;
+      }
+      for (int i = 0; i < num; i++) { // For each row of cells
+        int k = 0;
+        int numvalid = 0;
+        for (int j = 0; j < numCells; j++) { // Scan the cell list for matches for this row.
+          if (meanMask->data.PS_TYPE_VECTOR_MASK_DATA[j] & PM_PATTERN_IGNORE) {
+            continue;
+          }
+          if (!(meanMask->data.PS_TYPE_VECTOR_MASK_DATA[j] & PM_PATTERN_TWEAK)) {
+            continue;
+          }
+
+          pmCell *cell = chip->cells->data[j];
+          const char *cellName = psMetadataLookupStr(NULL,cell->concepts, "CELL.NAME");
+          int x = cellName[2] - '0';
+          int y = cellName[3] - '0';
+          if (((mode == 0)&&(doRows)&&(y == i))||
+              ((mode == 1)&&(doCols)&&(x == i))) {
+            inputs->data.S32[k] = j; // Add the index to the list of inputs.
+            k++;
+            numvalid++;
+          }
+        }
+        // Allocate the median image and mask
+        pmCell    *exCell  = chip->cells->data[inputs->data.S32[0]];
+        pmReadout *example = exCell->readouts->data[0];
+        pmReadout *median = pmReadoutAlloc(NULL);
+        median->image = psImageAlloc(example->image->numCols,
+                                     example->image->numRows,
+                                     PS_TYPE_F32);
+        median->mask = psImageAlloc(example->image->numCols,
+                                     example->image->numRows,
+                                    PS_TYPE_IMAGE_MASK);
+        // Block to do median.
+        {
+          psImageMaskType badMask = 0x7F;  // Configuable would be better.
+          for (int y = 0 ; y < example->image->numRows; y++) {
+            for (int x = 0; x < example->image->numCols; x++) {
+              int badVal = 0;
+              for (k = 0; k < numvalid; k++) {
+                pmCell *cell   = chip->cells->data[inputs->data.S32[k]];
+                pmReadout *ro  = cell->readouts->data[0];
+                psImage *image = ro->image;
+                psImage *mask  = ro->mask;
+                
+                pixelData->data.F32[k] = image->data.F32[y][x];
+                pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[k] = 0;
+                //#if (0)
+                double cell_median = cellMedians->data.F32[inputs->data.S32[k]];
+                double cell_sigma  = cellSigmas->data.F32[inputs->data.S32[k]];
+                if (fabs(pixelData->data.F32[k] - cell_median) > (3.0 * cell_sigma)) {
+                  pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[k] = 1;
+                  badVal++;
+                }
+                //#endif
+                if (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & badMask) {
+                  pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[k] = 1;
+                  badVal++;
+                }
+              }
+              for (k = numvalid; k < num; k++) {
+                pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[k] = 1;
+                badVal++;
+              }
+              if (badVal >= num) {
+                median->image->data.F32[y][x] = NAN;
+              }
+              else {
+                if (!psVectorStats(stats,pixelData,NULL,pixelMask,1)) {
+                  // Die gracefully.
+                  return(false);
+                }
+                int worst_k = -1;
+                double worst_v = 0;
+                for (k = 0; k < num; k++) {
+                  if (pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[k] == 0) {
+                    double v = fabs(stats->sampleMedian - pixelData->data.F32[k]);
+                    if (v > worst_v) {
+                      worst_v = v;
+                      worst_k = k;
+                    }
+                  }
+                }
+                if (worst_k != -1) {
+                  pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[worst_k] = 1;
+                  if (!psVectorStats(stats,pixelData,NULL,pixelMask,1)) {
+                    // Die gracefully.
+                    return(false);
+                  }
+                }
+                median->image->data.F32[y][x] = stats->sampleMedian;
+              }
+            }
+          }
+        } // End median block
+        
+        double correction_median = 0;
+        if (!psImageBackground(bgStats, NULL, median->image, NULL, maskVal, rng)) {
+          psWarning("Unable to measure background for cell section %d\n", i);
+          psErrorClear();
+        }
+        correction_median = bgStats->robustMedian;
+        for (int v = 0; v < median->image->numRows; v++) {
+          for (int u = 0; u < median->image->numCols; u++) {
+            median->image->data.F32[v][u] -= correction_median;
+          }
+        }
+
+        // Block to subtract median
+        for (int j = 0; j < numCells; j++) {
+          if (meanMask->data.PS_TYPE_VECTOR_MASK_DATA[j] & PM_PATTERN_IGNORE) {
+            continue;
+          }
+          if (!(meanMask->data.PS_TYPE_VECTOR_MASK_DATA[j] & PM_PATTERN_TWEAK)) {
+            continue;
+          }
+
+          pmCell *cell = chip->cells->data[j];
+
+          const char *cellName = psMetadataLookupStr(NULL,cell->concepts, "CELL.NAME");
+          int x = cellName[2] - '0';
+          int y = cellName[3] - '0';
+          if (((mode == 0)&&(doRows)&&(y == i))||
+              ((mode == 1)&&(doCols)&&(x == i))) {
+            psLogMsg("psModules.detrend",PS_LOG_DETAIL,"PATTERN.MEDIAN mode: %d Cell Statistics Pre: %d %d %d %s %g %g\n",
+                     mode, j,x,y,cellName,cellMedians->data.F32[j],cellSigmas->data.F32[j]);
+            pmReadout *ro = cell->readouts->data[0];
+            psImage *im = ro->image;
+            for (int v = 0; v < im->numRows; v++) {
+              for (int u = 0; u < im->numCols; u++) {
+                if (isfinite(median->image->data.F32[v][u])) {
+
+                  im->data.F32[v][u] -= (median->image->data.F32[v][u]);//  - correction_median);
+                }
+              }
+            }
+            if (!psImageBackground(bgStats, NULL, ro->image, ro->mask, maskVal, rng)) {
+              psWarning("Unable to measure background for cell %d\n", j);
+              psErrorClear();
+              meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PM_PATTERN_ERROR;
+              continue;
+            }
+            psLogMsg("psModules.detrend",PS_LOG_DETAIL,"PATTERN.MEDIAN mode: %d Cell Statistics Post: %d %d %d %s %g %g\n",
+                     mode, j,x,y,cellName,bgStats->robustMedian,bgStats->robustStdev);
+            cellMedians->data.F32[j] = bgStats->robustMedian;
+            cellSigmas->data.F32[j]  = bgStats->robustStdev;
+          }
+        } // End subtraction block
+        // Write out the median?
+#if (1)
+        const char *expname = psMetadataLookupStr(NULL,chip->hdu->header,"FILENAME");
+        char filename[256];
+        pmCell *exCell2 = chip->cells->data[inputs->data.S32[0]];
+        snprintf(filename,sizeof(filename),
+                 "/home/panstarrs/watersc1/pattern_median.20130201/tests/%s.%d.%d.fits",
+                 expname,mode,i);
+        psFits *fits = psFitsOpen(filename,"w");
+        if (!fits) {
+          fprintf(stderr,"Couldn't write\n");
+        }
+        if (!psFitsWriteImage(fits,exCell2->hdu->header,median->image,0,NULL)) {
+          fprintf(stderr,"Couldn't write2\n");
+        }
+        if (!psFitsClose(fits)) {
+          fprintf(stderr,"Couldn't write3\n");
+        }
+#endif
+        
+        psFree(median->image);
+        psFree(median->mask);
+        psFree(median);
+      }
+    }
+    psFree(bgStats);
+    psFree(rng);
+    psFree(cellMedians);
+    psFree(cellSigmas);
+    psFree(meanMask);
+    psFree(stats);
+    psFree(inputs);
+    psFree(pixelData);
+    psFree(pixelMask);
+
+
+    return true;
+}

branches/czw_branch/20130211/psModules/src/detrend/pmPattern.h

@@ -84 +84 @@
                         );
 
+/// Fix the pattern noise using medians
+bool pmPatternMedian(
+                     pmChip *chip,            ///< Chip to correct
+                     const psVector *tweak,   ///< U8 vector indicating whether to tweak the corresponding cell
+                     psStatsOptions bgStat,   ///< Statistic to use for background measurement
+                     psStatsOptions cellStat, ///< Statistic to use for combination of cell background measurementsn
+                     psImageMaskType maskVal, ///< Mask value to use
+                     psImageMaskType maskBad, ///< Mask value to give bad pixels
+                     bool doRows,             ///< Do correction in cell rows
+                     bool doCols              ///< Do correction in cell columns
+                     );
+
+