Index: trunk/ippTools/src/regtool.c
===================================================================
--- trunk/ippTools/src/regtool.c	(revision 8268)
+++ trunk/ippTools/src/regtool.c	(revision 8279)
@@ -12,9 +12,7 @@
 static bool updateexpMode(pxConfig *config);
 static bool updateimfileMode(pxConfig *config);
-static psArray *newFrameSearchPending(pxConfig *config);
-static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp);
+// static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp);
 static p2PendingExpRow *newToP2PendingExp(pxConfig *config, newExpRow *newExp);
-static p2PendingImfileRow *newImfileToP2PendingImfile(newImfileRow *newImfile);
-static rawScienceFrame *newToRawScienceFrame(pxConfig *config, newFrame *newFrame);
+static p2PendingImfileRow *rawImfileToP2PendingImfile(pxConfig *config, rawImfileRow *rawImfile);
 static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp);
 static rawDetrendExpRow *newToRawDetrendExp(pxConfig *config, newExpRow *exp);
@@ -179,4 +177,5 @@
         if (!status) {
             psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id");
+            psFree(query);
             return false;
         }
@@ -188,6 +187,8 @@
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
+        psFree(query);
+        return false;
+    }
+    psFree(query);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -205,5 +206,188 @@
     }
 
+    // start a transaction so we don't end up with an exp in
+    // rawScience/DetrendExp and in newExp
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(output);
+        return false;
+    }
+
+    // if it's a detrend exp
+    if (detrend) {        
+        for (long i = 0; psArrayLength(output) > i; i++) {
+            psMetadata *row = output->data[i];
+            // convert metadata into a newExp object
+            newExpRow *newExp = newExpObjectFromMetadata(row);
+            // convert newExp object into a rawDetrendExp object
+            rawDetrendExpRow *rawExp = newToRawDetrendExp(config, newExp);
+            if (!rawExp) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawDetrendExp");
+                psFree(newExp);
+                psFree(output);
+                return false;
+            }
+            // insert the rawDetrendExp object into the database
+            if (!rawDetrendExpInsertObject(config->dbh, rawExp)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(rawExp);
+                psFree(newExp);
+                psFree(output);
+                return false;
+            }
+            psFree(rawExp);
+            // delete the newExp object from the database
+            if (!newExpDeleteObject(config->dbh, newExp)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(newExp);
+                psFree(output);
+                return false;
+            }
+            psFree(newExp);
+        }
+    } 
+
+    // else
+    // it's a science exp
     for (long i = 0; psArrayLength(output) > i; i++) {
+        psMetadata *row = output->data[i];
+        // convert metadata into a newExp object
+        newExpRow *newExp = newExpObjectFromMetadata(row);
+        // convert newExp object into a rawDetrendExp object
+        rawScienceExpRow *rawExp = newToRawScienceExp(config, newExp);
+        if (!rawExp) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawScienceExp");
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        // insert the rawDetrendExp object into the database
+        if (!rawScienceExpInsertObject(config->dbh, rawExp)) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(rawExp);
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        psFree(rawExp);
+        // delete the newExp object from the database
+        if (!newExpDeleteObject(config->dbh, newExp)) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        // insert an entry into the p2PendingExp table
+        p2PendingExpRow *p2PendingExp = newToP2PendingExp(config, newExp);
+        if (!p2PendingExp) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to p2PendingExp");
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        // insert the p2PendingExp object into the database
+        if (!p2PendingExpInsertObject(config->dbh, p2PendingExp)) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(p2PendingExp);
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        psFree(p2PendingExp);
+        // find all of the rawImfiles associated with the p2PendingExp object
+        psArray *rawImfiles = NULL;
+        {
+            psMetadata *where = psMetadataAlloc();
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", newExp->exp_id)) {
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
+                psFree(where);
+                psFree(newExp);
+                psFree(output);
+                return false;
+            }
+            rawImfiles = rawImfileSelectRowObjects(config->dbh, where, 0);
+            psFree(where);
+        }
+        // sanity check the number of rawImfiles found
+        if (!rawImfiles) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "found no rawImfiles associated with exp_id %s", newExp->exp_id);
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        if (psArrayLength(rawImfiles) != newExp->imfiles) {
+            // rollback
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "found %d rawImfiles but expected %d",
+                    psArrayLength(rawImfiles), newExp->imfiles);
+            psFree(newExp);
+            psFree(output);
+            return false;
+        }
+        // convert the rawImfiles into p2PendingImfiles and insert them into
+        // the database
+        for (long i = 0; i < psArrayLength(rawImfiles); i++) {
+            rawImfileRow *rawImfile = rawImfiles->data[i];
+            p2PendingImfileRow *pendingImfile = rawImfileToP2PendingImfile(config, rawImfile);
+            if (!p2PendingImfileInsertObject(config->dbh, pendingImfile)){
+                // rollback
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(pendingImfile);
+                psFree(rawImfiles);
+                psFree(newExp);
+                psFree(output);
+            }
+            psFree(pendingImfile);
+        }
+        psFree(rawImfiles);
+        psFree(newExp);
+    }
+
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
     }
 
@@ -349,49 +533,5 @@
 }
 
-#if 0
-static psArray *newFrameSearchPending(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    char *query = "select newImfile.* from newImfile 
-        LEFT JOIN newExp USING(exp_id)
-        LEFT JOIN rawScienceExp USING(exp_id)
-        LEFT JOIN rawDetrendExp USING (exp_id)
-        WHERE newExp.exp_id is NOT NULL 
-        AND rawScienceExp.exp_id IS NULL
-        AND rawDetrendExp.exp_id IS NULL";
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    psArray *newExp = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        // XXX check psError here
-        psError(PS_ERR_UNKNOWN, false, "failed to fetch the database result");
-        return false;
-    }
-
-    for (long i = 0; i < psArrayLength(newExp); i++) {
-        newExpRow *exp = newExp->data[i];
-        psMetadata *where = psMetadataAlloc();
-        if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, NULL, exp->exp_id)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
-            psFree(where);
-            psFree(newExp);
-            return NULL;
-        }
-        new
-
-        psFree(where);
-    }
-
-
-    return new;
-}
-
-#endif
-
+# if 0
 static p1PendingExpRow *newToP1PendingExp(newExpRow *newExp)
 {
@@ -412,4 +552,5 @@
     );
 }
+#endif
 
 static p2PendingExpRow *newToP2PendingExp(pxConfig *config, newExpRow *exp)
@@ -418,72 +559,6 @@
     PS_ASSERT_PTR_NON_NULL(exp, NULL);
 
-    bool status = false;
-    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
-        return false;
-    }
-    if (!filter) {
-        psError(PS_ERR_UNKNOWN, true, "-filter is required");
-        return false;
-    }
-    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
-        return false;
-    }
-    if (isnan(airmass)) {
-        psError(PS_ERR_UNKNOWN, true, "-airmass is required");
-        return false;
-    }
-    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
-        return false;
-    }
-    if (isnan(ra)) {
-        psError(PS_ERR_UNKNOWN, true, "-ra is required");
-        return false;
-    }
-    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
-        return false;
-    }
-    if (isnan(decl)) {
-        psError(PS_ERR_UNKNOWN, true, "-decl is required");
-        return false;
-    }
-    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
-        return false;
-    }
-    if (isnan(exp_time)) {
-        psError(PS_ERR_UNKNOWN, true, "-exp_time is required");
-        return false;
-    }
-    psF64 background = psMetadataLookupF64(&status, config->args, "-background");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -background");
-        return false;
-    }
-    if (isnan(background)) {
-        psError(PS_ERR_UNKNOWN, true, "-background is required");
-        return false;
-    }
-
     return p2PendingExpRowAlloc(
         exp->exp_id,
-        exp->camera,
-        exp->telescope,
-        exp->exp_type,
-        exp->imfiles,
-        filter,
-        airmass,
-        ra,
-        decl,
-        exp_time,
-        background,
         "my recipe",
         0xff, // XXX calc version number
@@ -492,10 +567,10 @@
 }
 
-static p2PendingImfileRow *newImfileToP2PendingImfile(newImfileRow *newImfile)
+static p2PendingImfileRow *rawImfileToP2PendingImfile(pxConfig *config, rawImfileRow *rawImfile)
 {
     return p2PendingImfileRowAlloc(
-        newImfile->exp_id,
-        newImfile->class_id,
-        newImfile->uri,
+        rawImfile->exp_id,
+        rawImfile->class_id,
+        rawImfile->uri,
         "my recipe",
         0xff, // XXX calc version number
@@ -504,40 +579,4 @@
 }
 
-#if 0
-static rawScienceFrame *newToRawScienceFrame(pxConfig *config, newFrame *newFrame) 
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-    PS_ASSERT_PTR_NON_NULL(newFrame, NULL);
-
-    newExpRow *newExp = newFrame->exposure;
-    rawScienceExpRow *rawScienceExp = newToRawScienceExp(config, newExp);
-    if (!rawScienceExp) {
-        psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawScienceExp");
-        return NULL;
-    }
-
-    psArray *newImages = newFrame->images;
-    psArray *rawImages = psArrayAlloc(psArrayLength(newImages));
-    for (long i = 0; i < psArrayLength(newImages); i++) {
-        newImfileRow *newImfile = newImages->data[i];
-        rawImfileRow *rawImfile = rawImfileRowAlloc(
-            newImfile->exp_id,
-            newImfile->class,
-            newImfile->class_id,
-            newImfile->uri
-        );
-        psArrayAdd(rawImages, 0, rawImfile);
-        psFree(rawImfile);
-    }
-
-    rawScienceFrame *rawScienceFrame = rawScienceFrameAlloc(rawScienceExp, rawImages);
-    psFree(rawScienceExp);
-    psFree(rawImages);
-
-    return rawScienceFrame;
-}
-
-#endif
-
 static rawScienceExpRow *newToRawScienceExp(pxConfig *config, newExpRow *exp)
 {
@@ -545,73 +584,7 @@
     PS_ASSERT_PTR_NON_NULL(exp, NULL);
 
-    bool status = false;
-    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
-        return NULL;
-    }
-    if (!filter) {
-        psError(PS_ERR_UNKNOWN, true, "-filter is required");
-        return NULL;
-    }
-    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
-        return NULL;
-    }
-    if (isnan(airmass)) {
-        psError(PS_ERR_UNKNOWN, true, "-airmass is required");
-        return NULL;
-    }
-    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
-        return NULL;
-    }
-    if (isnan(ra)) {
-        psError(PS_ERR_UNKNOWN, true, "-ra is required");
-        return NULL;
-    }
-    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
-        return NULL;
-    }
-    if (isnan(decl)) {
-        psError(PS_ERR_UNKNOWN, true, "-decl is required");
-        return NULL;
-    }
-    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
-        return NULL;
-    }
-    if (isnan(exp_time)) {
-        psError(PS_ERR_UNKNOWN, true, "-exp_time is required");
-        return NULL;
-    }
-    psF64 background = psMetadataLookupF64(&status, config->args, "-background");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -background");
-        return NULL;
-    }
-    if (isnan(background)) {
-        psError(PS_ERR_UNKNOWN, true, "-background is required");
-        return NULL;
-    }
-
-    return rawScienceExpRowAlloc(
-        exp->exp_id,
-        exp->camera,
-        exp->telescope,
-        exp->exp_type,
-        exp->imfiles,
-        filter,
-        airmass,
-        ra,
-        decl,
-        exp_time,
-        background
-    );
+    // XXX this is dangerous but we should be able to get away with this as
+    // long rawScienceExp & rawDetrendExp are idetnical
+    return (rawScienceExpRow *)newToRawDetrendExp(config, exp);
 }
 
@@ -667,11 +640,65 @@
         return false;
     }
-    psF64 background = psMetadataLookupF64(&status, config->args, "-background");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -background");
-        return false;
-    }
-    if (!exp_time) {
-        psError(PS_ERR_UNKNOWN, true, "-background is required");
+    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
+        return false;
+    }
+    if (isnan(bg)) {
+        psError(PS_ERR_UNKNOWN, true, "-bg is required");
+        return false;
+    }
+    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
+        return false;
+    }
+    if (isnan(bg_stdev)) {
+        psError(PS_ERR_UNKNOWN, true, "-bg_stdev is required");
+        return false;
+    }
+    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
+        return false;
+    }
+    if (isnan(bg_mean_stdev)) {
+        psError(PS_ERR_UNKNOWN, true, "-bg_mean_stdev is required");
+        return false;
+    }
+    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
+        return false;
+    }
+    if (isnan(alt)) {
+        psError(PS_ERR_UNKNOWN, true, "-alt is required");
+        return false;
+    }
+    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
+        return false;
+    }
+    if (isnan(az)) {
+        psError(PS_ERR_UNKNOWN, true, "-az is required");
+        return false;
+    }
+    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
+        return false;
+    }
+    if (isnan(ccd_temp)) {
+        psError(PS_ERR_UNKNOWN, true, "-ccd_temp is required");
+        return false;
+    }
+    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");
+        return false;
+    }
+    if (isnan(posang)) {
+        psError(PS_ERR_UNKNOWN, true, "-posang is required");
         return false;
     }
@@ -688,5 +715,11 @@
         decl,
         exp_time,
-        background
+        bg,
+        bg_stdev,
+        bg_mean_stdev,
+        alt,
+        az,
+        ccd_temp,
+        posang
     );
 }
@@ -752,5 +785,5 @@
         return false;
     }
-    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg");
+    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
@@ -761,5 +794,5 @@
         return false;
     }
-    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg");
+    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
@@ -770,5 +803,5 @@
         return false;
     }
-    psF64 alt = psMetadataLookupF64(&status, config->args, "-bg");
+    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
@@ -779,5 +812,5 @@
         return false;
     }
-    psF64 az = psMetadataLookupF64(&status, config->args, "-bg");
+    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
@@ -788,5 +821,5 @@
         return false;
     }
-    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-bg");
+    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
@@ -797,5 +830,5 @@
         return false;
     }
-    psF64 posang = psMetadataLookupF32(&status, config->args, "-bg");
+    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");