Index: trunk/ippTools/src/pzgetexp.c
===================================================================
--- trunk/ippTools/src/pzgetexp.c	(revision 9967)
+++ trunk/ippTools/src/pzgetexp.c	(revision 9968)
@@ -43,6 +43,10 @@
     // a metadatadb function to retrun the last entry without removing it from
     // the database
-    psArray *summitExps = summitExpSelectRowObjects(config->dbh, NULL, 0);
-    if (summitExps) {
+    psArray *summitExps = summitExpSelectRowObjects(config->dbh, NULL, 1);
+    if (!summitExps) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        goto FAIL;
+    }
+    if (psArrayLength(summitExps)) {
         haveLastFileSet = true;
         summitExpRow *lastExps = summitExps->data[psArrayLength(summitExps) - 1];
@@ -83,30 +87,54 @@
         goto FAIL;
     }
-
-    // try not to insert duplicate summitExp entries
-    // XXX this will become very expensive as the number of summitExp entry
-    // grows -- is just blindly ignoring database errors the best thing to do?
-    if (summitExps) {
-        for (long i = 0; i < psArrayLength(newSummitExps); i++) {
-            summitExpRow *newSummitExp = newSummitExps->data[i];
-            for (long j = 0; j < psArrayLength(summitExps); j++) {
-                summitExpRow *summitExp = summitExps->data[j];
-                if (strcmp(newSummitExp->exp_id,
-                           summitExp->exp_id) == 0) {
-                    psArrayRemoveData(newSummitExps, newSummitExp);
-                    // dec the counter as the array just got shorter
-                    // and we don't want to skip elemnts
-                    i--;
-                    break;
-                }
-            }
-        }
-        psFree(summitExps);
-    }
-
-    // insert new entries into the database
-    for (long i = 0; i < psArrayLength(newSummitExps); i++) {
-        if (!summitExpInsertObject(config->dbh, newSummitExps->data[i])) {
-            psError(PS_ERR_UNKNOWN, false, "dbh access failed");
+    if (!psArrayLength(newSummitExps)) {
+        psError(PS_ERR_UNKNOWN, true, "no new fileSet/exp IDs");
+        psFree(newSummitExps);
+        return true;
+    }
+
+    // create a temporry table
+    {
+        char *query = "CREATE TEMPORARY TABLE incoming" 
+            " (exp_id VARCHAR(64), camera VARCHAR(64), telescope VARCHAR(64), exp_type VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(exp_id, camera, telescope))"
+           " ENGINE=MEMORY";
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(newSummitExps);
+            goto FAIL;
+        }
+    }
+
+    {
+        char *query = "INSERT INTO incoming (exp_id, camera, telescope, exp_type, uri) VALUES (?, ?, ?, ?, ?)";
+
+        long inserted = p_psDBRunQueryPrepared(config->dbh, newSummitExps, query);
+        if (inserted < 0) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(newSummitExps);
+            goto FAIL;
+        }
+        // sanity check that we actually inserted something
+        if (inserted == 0) {
+            psError(PS_ERR_UNKNOWN, false, "database error -- we should have inserted at least one row");
+            psFree(newSummitExps);
+            goto FAIL;
+        }
+    }
+
+    psFree(newSummitExps);
+
+    {
+        char *query = 
+            "INSERT INTO summitExp" 
+            "   SElECT incoming.* FROM incoming"
+            "   LEFT JOIN summitExp"
+            "       USING(exp_id, camera, telescope)"
+            "   WHERE"
+            "       summitExp.exp_id is NULL"
+            "       AND summitExp.camera is NULL"
+            "       AND summitExp.telescope is NULL";
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             goto FAIL;
         }
@@ -127,4 +155,18 @@
     PS_ASSERT_PTR_NON_NULL(config, NULL);
     PS_ASSERT_PTR_NON_NULL(str, NULL);
+    
+    // these are constants for all records parsed -- look them up before we do
+    // any work
+    bool status = false;
+    char *camera = psMetadataLookupStr(&status, config->args, "-inst");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup item '-inst'");
+        return NULL;
+    }
+    char *telescope = psMetadataLookupStr(&status, config->args, "-telescope");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup item '-telescope'");
+        return NULL;
+    }
 
     psList *doc = psStringSplit(str, "\n", false);
@@ -155,4 +197,5 @@
         }
 
+        // find the values of interest
         psListIterator *tokenCursor = psListIteratorAlloc(tokens, 0, false);
         char *uri       = psListGetAndIncrement(tokenCursor);
@@ -163,22 +206,47 @@
         if (time) {} // silence unused warning
 
-        bool status = false;
-        char *camera_name = psMetadataLookupStr(&status, config->args,
-            "-inst");
-        char *telescope   = psMetadataLookupStr(&status, config->args,
-            "-telescope");
-
-        summitExpRow *row = summitExpRowAlloc(
-            exp_id,
-            camera_name,
-            telescope,
-            exp_type,
-            uri
-        );
+        // create a new metadata to represent this line and it's values
+        psMetadata *md = psMetadataAlloc();
+        if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_id", 0, NULL, exp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
+            psFree(md);
+            psFree(tokenCursor);
+            psFree(tokens);
+            return NULL;
+        }
+        if (!psMetadataAddStr(md, PS_LIST_TAIL, "camera", 0, NULL, camera)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item camera");
+            psFree(md);
+            psFree(tokenCursor);
+            psFree(tokens);
+            return NULL;
+        }
+        if (!psMetadataAddStr(md, PS_LIST_TAIL, "telescope", 0, NULL, telescope)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item telescope");
+            psFree(md);
+            psFree(tokenCursor);
+            psFree(tokens);
+            return NULL;
+        }
+        if (!psMetadataAddStr(md, PS_LIST_TAIL, "exp_type", 0, NULL, exp_type)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item exp_type");
+            psFree(md);
+            psFree(tokenCursor);
+            psFree(tokens);
+            return NULL;
+        }
+        if (!psMetadataAddStr(md, PS_LIST_TAIL, "uri", 0, NULL, uri)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item uri");
+            psFree(md);
+            psFree(tokenCursor);
+            psFree(tokens);
+            return NULL;
+        }
+
+        psArrayAdd(summitExps, 0, md);
 
         psFree(tokenCursor);
         psFree(tokens);
 
-        psArrayAdd(summitExps, 0, row);
     }