Index: /trunk/archive/dettools/doc/dettools.txt
===================================================================
--- /trunk/archive/dettools/doc/dettools.txt	(revision 5870)
+++ /trunk/archive/dettools/doc/dettools.txt	(revision 5871)
@@ -30,24 +30,33 @@
 
 dettool -select raw [-state state] [-outmode mode]
- * select the frames to be processed and extract the list of files 
+ * select the unprocessed input infiles
  - output of this program is used by pantasks to schedule the ppImage
  run on each image
 
 dettool -select stack [-state state] [-outmode mode]
- * select the files to be stacked for a given detRun
+ * select the files to be stacked for a given detRun, if exposures are ready
  - output of this program is used by ppMerge to build a master stack
 
-dettool -select residual [-state state]
- * select the frames to be used to generate residuals
+dettool -select rawresid [-state state]
+ * select the residual imfiles to be generated
  - output of this program is used for another ppImage run 
 
-detmerge 
+dettool -update raw
+ * select the raw input exposures, count processed imfiles, update if done
+
+dettool -update resid
+ * select the resid exposures, count processed resid imfiles, update if done
+ - for a given resid exposure, compile the results from the stacks for
+ each chip and renormalize the output files as needed.
+
+dettool -select norm
+ * select the master detrend frames & imfiles to be normalized
+
+dettool -resid
+ * construct the resid exposures and imfiles given 
+
+dettool -assess
  * for a given master detrend frame, compile the results from the
- stacks for each chip and renormalize the output files as needed.
- assemble the resulting master stats as available
-
-detstats
- * for a given master detrend frame, compile the results from the
- residual images and assess the validity of the input images and of
+ residual exposures and assess the validity of the input exposures and of
  the complete stack.  if too many input images are rejected, the
  affect the master detrend frame state.  if input images are rejected
@@ -66,2 +75,39 @@
    after process that determines which happens next.
 
+
+
+States:
+ input detrend exposures:
+  RAW
+  PROCESSED
+
+ input detrend imfiles:
+  RAW
+  PROCESSED
+  
+ master detrend frames
+  RAW
+  NORMALIZED
+  MKRESID
+  SUCCESS
+  FAILURE
+  RETRY
+  (also has NEW/PRIOR flag)
+
+ master detrend imfiles
+  NEW (not yet created)
+  RAW (created, but not normalized)
+  NORMALIZED
+
+ resid exposure
+  RAW
+  PROCESSED
+  ASSESSED
+
+ resid imfiles:
+  RAW
+  PROCESSED
+
+ master detrend run:
+  NEW
+  DONE
Index: /trunk/archive/dettools/src/detSelectInputImfiles.c
===================================================================
--- /trunk/archive/dettools/src/detSelectInputImfiles.c	(revision 5871)
+++ /trunk/archive/dettools/src/detSelectInputImfiles.c	(revision 5871)
@@ -0,0 +1,159 @@
+# include "dettools.h"
+
+// select raw frames (exposure+images) which match the given config options
+psArray *detSelectInputImfiles (detConfig *config, int state) {
+
+    // build 'where' structure
+    psMetadata *where = psMetadataAlloc ();
+
+    psMetadataAddStr (where, PS_LIST_TAIL, "STATE", 0, "==", state);
+
+    // XXX EAM : do we really want/need to allow these restrictions?
+    if (config->type != DET_TYPE_UNDEF) {
+	psMetadataAddU8  (where, PS_LIST_TAIL, "TYPE",  0, "==", config->type);
+    }
+    if (config->camera != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "CAMERA", 0, "==", config->camera);
+    }
+    if (config->filter != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "FILTER", 0, "==", config->filter);
+    }
+    if (config->timeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_START", 0, "<", config->timeStop);
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_STOP", 0, ">", config->timeStart);
+    }
+    if (config->exptimeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, "<", config->exptimeMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, ">", config->exptimeMax);
+    }
+    if (config->airmassSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, "<", config->airmassMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, ">", config->airmassMax);
+    }
+
+    psArray *imfiles = detInputImfileSelectRows (config->database, where, MAX_ROWS);
+    psFree (where);
+
+    return imfiles;
+} 
+
+// select raw frames (exposure+images) which match the given config options
+psArray *detSelectRawUpdateExposures (detConfig *config) {
+
+    // build 'where' structure
+    psMetadata *where = psMetadataAlloc ();
+
+    psMetadataAddStr (where, PS_LIST_TAIL, "STATE", 0, "==", DET_STATE_RAW);
+
+    // XXX EAM : do we really want/need to allow these restrictions?
+    if (config->type != DET_TYPE_UNDEF) {
+	psMetadataAddU8  (where, PS_LIST_TAIL, "TYPE",  0, "==", config->type);
+    }
+    if (config->camera != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "CAMERA", 0, "==", config->camera);
+    }
+    if (config->filter != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "FILTER", 0, "==", config->filter);
+    }
+    if (config->timeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_START", 0, "<", config->timeStop);
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_STOP", 0, ">", config->timeStart);
+    }
+    if (config->exptimeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, "<", config->exptimeMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, ">", config->exptimeMax);
+    }
+    if (config->airmassSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, "<", config->airmassMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, ">", config->airmassMax);
+    }
+
+    psArray *exposures = detInputExposureSelectRows (config->database, where, MAX_ROWS);
+    psFree (where);
+
+    // output array of detMasterExposures
+    psArray *ready = psArrayAlloc (exposures->n);
+    ready->n = 0;
+
+    // 'where' to select each exposure
+    psMetadata *where = psMetadataAlloc ();
+    for (int i = 0; i < exposures->n; i++) {
+	detInputExposure *exposure = exposures->data[i];
+
+	// are all chips finished processing?
+	psMetadataAddStr (where, PS_LIST_TAIL, "STATE", PS_META_REPLACE, "==", DET_STATE_PROCESSED);
+	psMetadataAddStr (where, PS_LIST_TAIL, "DET_ID", PS_META_REPLACE, "==", exposure->detID);
+	psMetadataAddStr (where, PS_LIST_TAIL, "VERSION", PS_META_REPLACE, "==", exposure->version);
+
+	psArray *imfiles = detInputImfileSelectRows (config->database, where, MAX_ROWS);
+	if (imfiles->n != exposure->Nclass) {
+	    psFree (imfiles);
+	    continue;
+	}
+
+	psFree (imfiles);
+	psArrayAdd (ready, 100, exposure);
+    }
+    return ready;
+} 
+
+// select raw frames (exposure+images) which match the given config options
+psArray *detSelectResidUpdateExposures (detConfig *config) {
+
+    // build 'where' structure
+    psMetadata *where = psMetadataAlloc ();
+
+    psMetadataAddStr (where, PS_LIST_TAIL, "STATE", 0, "==", DET_STATE_RAW);
+
+    // XXX EAM : do we really want/need to allow these restrictions?
+    if (config->type != DET_TYPE_UNDEF) {
+	psMetadataAddU8  (where, PS_LIST_TAIL, "TYPE",  0, "==", config->type);
+    }
+    if (config->camera != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "CAMERA", 0, "==", config->camera);
+    }
+    if (config->filter != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "FILTER", 0, "==", config->filter);
+    }
+    if (config->timeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_START", 0, "<", config->timeStop);
+	psMetadataAddTime (where, PS_LIST_TAIL, "TIME_STOP", 0, ">", config->timeStart);
+    }
+    if (config->exptimeSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, "<", config->exptimeMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "EXPTIME", 0, ">", config->exptimeMax);
+    }
+    if (config->airmassSelect) {
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, "<", config->airmassMin);
+	psMetadataAddTime (where, PS_LIST_TAIL, "AIRMASS", 0, ">", config->airmassMax);
+    }
+
+    psArray *exposures = detResidExposureSelectRows (config->database, where, MAX_ROWS);
+    psFree (where);
+
+    // output array of detMasterExposures
+    psArray *ready = psArrayAlloc (exposures->n);
+    ready->n = 0;
+
+    // 'where' to select each exposure
+    psMetadata *where = psMetadataAlloc ();
+    for (int i = 0; i < exposures->n; i++) {
+	detResidExposure *exposure = exposures->data[i];
+
+	// are all chips finished processing?
+	psMetadataAddStr (where, PS_LIST_TAIL, "STATE", PS_META_REPLACE, "==", DET_STATE_PROCESSED);
+	psMetadataAddStr (where, PS_LIST_TAIL, "DET_ID", PS_META_REPLACE, "==", exposure->detID);
+	psMetadataAddStr (where, PS_LIST_TAIL, "VERSION", PS_META_REPLACE, "==", exposure->version);
+	psMetadataAddStr (where, PS_LIST_TAIL, "ITERATION", PS_META_REPLACE, "==", exposure->iteration);
+
+	psArray *imfiles = detResidImfileSelectRows (config->database, where, MAX_ROWS);
+	if (imfiles->n != exposure->Nclass) {
+	    psFree (imfiles);
+	    continue;
+	}
+
+	psFree (imfiles);
+	psArrayAdd (ready, 100, exposure);
+    }
+    return ready;
+} 
Index: /trunk/archive/dettools/src/detSelectMasterExposures.c
===================================================================
--- /trunk/archive/dettools/src/detSelectMasterExposures.c	(revision 5871)
+++ /trunk/archive/dettools/src/detSelectMasterExposures.c	(revision 5871)
@@ -0,0 +1,138 @@
+# include "dettools.h"
+
+// select raw frames (exposure+images) which match the given config options
+psArray *detSelectMasterExposuresForStack (detConfig *config) {
+
+    // build 'where' structure
+    psMetadata *where = psMetadataAlloc ();
+
+    psMetadataAddStr (where, PS_LIST_TAIL, "STATE", 0, "==", DET_STATE_RAW);
+
+    // XXX EAM : do we really want/need to allow these restrictions?
+    if (config->type != DET_TYPE_UNDEF) {
+	psMetadataAddU8  (where, PS_LIST_TAIL, "TYPE",  0, "==", config->type);
+    }
+    if (config->camera != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "CAMERA", 0, "==", config->camera);
+    }
+    if (config->filter != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "FILTER", 0, "==", config->filter);
+    }
+
+    psArray *exposures = detMasterExposureSelectRows (config->database, where, MAX_ROWS);
+    psFree (where);
+
+    // output array of detMasterExposures
+    psArray *ready = psArrayAlloc (exposures->n);
+    ready->n = 0;
+
+    // 'where' to select each exposure
+    psMetadata *where = psMetadataAlloc ();
+    for (int i = 0; i < exposures->n; i++) {
+	detMasterExposure *exposure = exposures->data[i];
+	
+	// is the input data for this master exposure ready for stacking?
+	psMetadataAddStr (where, PS_LIST_TAIL, "STATE", PS_META_REPLACE, "==", DET_STATE_PROCESSED);
+	psMetadataAddStr (where, PS_LIST_TAIL, "DET_ID", PS_META_REPLACE, "==", exposure->detID);
+	psMetadataAddStr (where, PS_LIST_TAIL, "VERSION", PS_META_REPLACE, "==", exposure->version);
+
+	psArray *inputs = detInputExposureSelectRows (config->database, where, MAX_ROWS);
+	if (inputs->n != exposure->nInput) {
+	    psFree (inputs);
+	    continue;
+	}
+
+	psFree (inputs);
+	psArrayAdd (ready, 100, exposure);
+    }
+    return ready;
+} 
+
+// select stack frames (detMasterImfile + detInputImfiles) which match the given master detrend exposures
+// XXX an alternative to this would construct the imfiles here based on the camera metadata
+psArray *detSelectMasterImfilesForStack (detConfig *config, psArray *exposures) {
+
+    psArray *frames = psArrayAlloc (100);
+
+    // 'where' to select each exposure
+    psMetadata *where = psMetadataAlloc ();
+    for (int i = 0; i < exposures->n; i++) {
+	detMasterExposure *exposure = exposures->data[i];
+	
+	// select the detrend master imfiles for this master exposure
+	psMetadataAddStr (where, PS_LIST_TAIL, "DET_ID", PS_META_REPLACE, "==", exposure->detID);
+	psMetadataAddStr (where, PS_LIST_TAIL, "VERSION", PS_META_REPLACE, "==", exposure->version);
+	psMetadataAddStr (where, PS_LIST_TAIL, "ITERATION", PS_META_REPLACE, "==", exposure->iteration);
+
+	psArray *detImfiles = detMasterImfileSelectRows (config->database, where, MAX_ROWS);
+
+	for (int j = 0; j < detImfiles->n; j++) {
+	    
+	    detMasterImfile *detImfile = detImfiles->data[j];
+
+	    // XXX : confirm that this detImfile has state DET_STATE_NEW ??
+
+	    // select the input imfiles for this master imfile
+	    psMetadataAddStr (where, PS_LIST_TAIL, "CLASS_ID", PS_META_REPLACE, "==", detImfile->classID);
+
+	    psArray *inputImfiles = detInputImfileSelectRows (config->database, where, MAX_ROWS);
+
+	    detStackFrame *stackFrame = detStackFrameAlloc (detImfile, inputImfiles);
+
+	    psArrayAdd (frames, 100, stackFrame);
+	}
+    }
+    return frames;
+} 
+
+
+// ************ normalization (put in new file) *******
+
+// select raw frames (exposure+images) which match the given config options
+psArray *detSelectMasterExposuresForNormalization (detConfig *config) {
+
+    // build 'where' structure
+    psMetadata *where = psMetadataAlloc ();
+
+    psMetadataAddStr (where, PS_LIST_TAIL, "STATE", 0, "==", DET_STATE_RAW);
+
+    // XXX EAM : do we really want/need to allow these restrictions?
+    if (config->type != DET_TYPE_UNDEF) {
+	psMetadataAddU8  (where, PS_LIST_TAIL, "TYPE",  0, "==", config->type);
+    }
+    if (config->camera != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "CAMERA", 0, "==", config->camera);
+    }
+    if (config->filter != NULL) {
+	psMetadataAddStr (where, PS_LIST_TAIL, "FILTER", 0, "==", config->filter);
+    }
+
+    psArray *exposures = detMasterExposureSelectRows (config->database, where, MAX_ROWS);
+    psFree (where);
+
+    // output array of detMasterExposures
+    psArray *frames = psArrayAlloc (exposures->n);
+    frames->n = 0;
+
+    // 'where' to select each exposure
+    psMetadata *where = psMetadataAlloc ();
+    for (int i = 0; i < exposures->n; i++) {
+	detMasterExposure *exposure = exposures->data[i];
+	
+	// is the input data for this master exposure ready for stacking?
+	psMetadataAddStr (where, PS_LIST_TAIL, "STATE", PS_META_REPLACE, "==", DET_STATE_RAW);
+	psMetadataAddStr (where, PS_LIST_TAIL, "DET_ID", PS_META_REPLACE, "==", exposure->detID);
+	psMetadataAddStr (where, PS_LIST_TAIL, "VERSION", PS_META_REPLACE, "==", exposure->version);
+	psMetadataAddStr (where, PS_LIST_TAIL, "ITERATION", PS_META_REPLACE, "==", exposure->iteration);
+
+	psArray *imfiles = detMasterImfileSelectRows (config->database, where, MAX_ROWS);
+	if (imfiles->n != exposure->nImfiles) {
+	    psFree (imfiles);
+	    continue;
+	}
+
+	detNormFrame *frame = detNormFrameAlloc (exposure, imfiles);
+	psArrayAdd (frames, 100, frame);
+    }
+    return frames;
+} 
Index: /trunk/archive/dettools/src/dettools.c
===================================================================
--- /trunk/archive/dettools/src/dettools.c	(revision 5871)
+++ /trunk/archive/dettools/src/dettools.c	(revision 5871)
@@ -0,0 +1,78 @@
+# include "p2search.h"
+
+int main (int argc, char **argv) {
+
+    psArray *imfiles;
+    psArray *frames;
+    psArray *pendingFrames;
+
+    dettoolsConfig (&config, argc, argv);
+
+    // -select options
+    if (config->mode == DET_MODE_SELECT_RAW) {
+	imfiles = detSelectInputImfiles (config, DET_STATE_RAW);
+	detWriteInputImfiles (config, imfiles);
+    }	
+
+    if (config->mode == DET_MODE_SELECT_RESID) {
+	imfiles = detSelectResidImfiles (config, DET_STATE_RAW);
+	detWriteResidImfiles (config, imfiles);
+    }	
+
+    if (config->mode == DET_MODE_SELECT_STACK) {
+	exposures = detSelectMasterExposuresForStack (config);
+	frames = detSelectMasterImfilesForStack (config, exposures);
+	detWriteStackFrames (config, frames);
+    }	
+
+    if (config->mode == DET_MODE_SELECT_NORM) {
+	frames  = detSelectMasterExposuresForNorm (config);
+	detWriteNormFrames (config, frames);
+    }	
+
+    // -update options
+    if (config->mode == DET_MODE_UPDATE_RAW) {
+	exposures = detSelectRawUpdateExposures (config);
+	detUpdateRawExposures (config, exposures);
+    }
+
+    if (config->mode == DET_MODE_UPDATE_RESID) {
+	exposures = detSelectResidUpdateExposures (config);
+	detUpdateResdiExposures (config, exposures);
+    }
+
+    if (config->mode == DET_MODE_UPDATE_RUN) {
+	// select master detrend exposures where (state == MKRESID)
+	// foreach master exposure:
+	//   select matching resid exposures where (state == PROCESSED)
+	//   if (residExposures->n != nInput) continue;
+	//   calculate summary stats
+	//   set state of masterExposure
+	//   create new masterExposure iteration (if needed)
+	//   set state of masterRun (if done)
+    }
+
+    // -define options
+    if (config->mode == DET_MODE_DEFINE_RUN) {
+	// select matching group, if desired
+	// select matching raw exposures, imfiles
+	// create masterRun based on options
+	// create new inputExposures
+	// create new inputImfiles
+	// create new detrendImfiles
+    }
+
+    if (config->mode == DET_MODE_DEFINE_RESID) {
+	// select master detrend exposures where (state == NORMALIZED)
+	// foreach master exposure:
+	//   select matching input exposures where (state == PROCESSED)
+	//   foreach input exposure
+	//     create matching resid exposure
+	//     select matching input imfiles
+	//     foreach input imfile
+	//       create matching resid imfile
+	//   set master exposure state to MKRESID
+    }
+
+    exit (0);
+}
Index: /trunk/archive/dettools/src/dettoolsConfig.c
===================================================================
--- /trunk/archive/dettools/src/dettoolsConfig.c	(revision 5871)
+++ /trunk/archive/dettools/src/dettoolsConfig.c	(revision 5871)
@@ -0,0 +1,68 @@
+# include "p2search.h"
+
+bool p2searchConfig (psConfig *config, int argc, char **argv) {
+
+    // Parse the configurations (re-org a la ppImage)
+    config->site = NULL;            // Site configuration
+    config->camera = NULL;          // Camera configuration
+    config->recipe = NULL;          // Recipe configuration
+    if (! pmConfigRead(&config->site, &config->camera, &config->recipe, &argc, argv, RECIPE)) {
+        psErrorStackPrint(stderr, "Can't find site configuration!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    // Parse other command-line arguments
+    config->arguments = psMetadataAlloc(); // The arguments, with default values
+
+    config->mode = P2_MODE_NONE;
+    if ((N = psArgumentGet (*argc, argv, "-quick"))) {
+	psArgumentRemove (N, argc, argv);
+	if (config->mode) psAbort ("p2search", "only one mode selection is allowed");
+	config->mode = P2_MODE_QUICK;
+    }
+    if ((N = psArgumentGet (*argc, argv, "-define"))) {
+	psArgumentRemove (N, argc, argv);
+	if (config->mode) psAbort ("p2search", "only one mode selection is allowed");
+	config->mode = P2_MODE_DEFINE;
+    }
+    if ((N = psArgumentGet (*argc, argv, "-pending"))) {
+	psArgumentRemove (N, argc, argv);
+	if (config->mode) psAbort ("p2search", "only one mode selection is allowed");
+	config->mode = P2_MODE_PENDING;
+    }
+    if ((N = psArgumentGet (*argc, argv, "-update"))) {
+	psArgumentRemove (N, argc, argv);
+	if (config->mode) psAbort ("p2search", "only one mode selection is allowed");
+	config->mode = P2_MODE_UPDATE;
+    }
+    if ((N = psArgumentGet (*argc, argv, "-done"))) {
+	psArgumentRemove (N, argc, argv);
+	if (config->mode) psAbort ("p2search", "only one mode selection is allowed");
+	config->mode = P2_MODE_UPDATE;
+    }
+
+    // paul's argument parsing convention requires: -key value 
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-quick",   0, "examine raw image table, write ppImage output", "");
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-define",  0, "examine raw image table, add to pending image table", "");
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-pending", 0, "examine pending image table, write ppImage output", "");
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-update",  0, "update pending image table", "");
+    psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-time",    0, "define time range of interest", "");
+    psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-camera",  0, "define camera of interest", "");
+    psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-filter",  0, "define filter of interest", "");
+
+    if (! psArgumentParse(config->arguments, &argc, argv) || argc != 2) {
+        printf("\nPan-STARRS Phase 2 Search Tool\n\n");
+        printf("Usage: %s [mode] [options]\n\n", argv[0]);
+        printf(" [mode] : -quick | -define | -pending | -update\n\n");
+        psArgumentHelp(config->arguments);
+        psFree(config->arguments);
+        exit(EXIT_FAILURE);
+    }
+
+    // add the input and output images to the arguments list
+    psMetadataAddStr (config->arguments, PS_LIST_TAIL, "-output", 0, "Name of the output image", argv[1]);
+
+    // define Database handle, if used
+    config->database = pmConfigDB(config->site);
+    return true;
+}