Index: trunk/ippToPsps/jython/detectionbatch.py
===================================================================
--- trunk/ippToPsps/jython/detectionbatch.py	(revision 35097)
+++ trunk/ippToPsps/jython/detectionbatch.py	(revision 35124)
@@ -389,14 +389,9 @@
         except: pass
        
-        # delete all detections with PSF_INST_MAG < -17.5, as decreed by Gene
+        # XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5)
+        # for examples of code which do this, go to r35097 or earlier
+
         BEFORE = self.scratchDb.getRowCount(ippTableName)
-        #don't do this anymore
-        #sql = "DELETE FROM " + ippTableName + " WHERE PSF_INST_MAG < -17.5"
-        #self.scratchDb.execute(sql)
-        #DELETED = BEFORE - self.scratchDb.getRowCount(ippTableName)
-        #results['SATDET'] = BEFORE - self.scratchDb.getRowCount(ippTableName)
-        #we aren't deletng these now
         results['SATDET'] = 0 
-        #self.logger.infoPair("Saturated detections", "%d deleted" % DELETED)
 
         # insert all detections into table
@@ -473,23 +468,10 @@
         self.scratchDb.execute(sql)
 
-        # now delete bad flux and bad chip positions
-        #sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux
-        #self.scratchDb.execute(sql)
-
         # add a instFlux = 0.0 -> 0.000001
+        # XXX EAM : why is this done?
         sql="UPDATE " + pspsTableName + " SET psfFlux = 0.0000001 WHERE psfFlux =  0"
         self.scratchDb.execute(sql)
       
-        # update cosmic ray and extended likelihoods
-        #sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0"
-        #self.scratchDb.execute(sql)
-        #sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0"
-        #self.scratchDb.execute(sql)
-
-        # remove detections will NULL inst flux or NULL peak ADU
-        # leave them in at this stage- don't delete them.
-        #results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux")
-        #results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU")
-        #we don't delete thos anymore
+        # we don't delete these anymore
         results['NULLINSTFLUX'] = 0;
         results['NULLPEAKADU'] = 0;
@@ -857,4 +839,6 @@
         self.logger.info("updated detectioncalibtable")
 
+        self.validChips.append(chipname)
+
         return True
 
@@ -865,4 +849,5 @@
 
         # loop through all OTAs again to update with DVO IDs
+        self.validChips = []    
         self.tablesToExport = []    
         self.tablesToExport.append("FrameMeta")
@@ -907,4 +892,7 @@
             return False
 
+        # XXX EAM : turn this on (and fix up the code) if we need it 
+        # self.removeDuplicateObjects()
+
         self.setMinMaxObjID(tables)
 
@@ -914,4 +902,47 @@
         
         return True
+
+    def removeDuplicateObjects(self):
+
+        sql = "DROP TABLE SkinnyObject_All"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            print "SkinnyObject_All table not yet defined"
+
+        sql = "CREATE TABLE SkinnyObject_All (objID bigint, ippObjID bigint, chipID char(16))"
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            print "failed to create table SkinnyObject_All"
+            raise
+
+        for chipname in self.validChips:
+            
+            sql = "INSERT INTO SkinnyObject_All (objID, ippObjID, chipID) \
+                   SELECT objID, ippObjID, '" + chipname + "' from SkinnyObject_" + chipname
+            try:
+                self.scratchDb.execute(sql)
+            except:
+                print "failed to insert entry SkinnyObject_All: ", chipname
+                raise
+            
+
+        Nduplicates = 0
+        sql = "SELECT n, objID from (select count(objID) as n, objID from SkinnyObject_All group by objID) as tmp where n > 1"
+        try:
+            rs = self.scratchDb.executeQuery(sql)
+            while (rs.next()):
+                objID = rs.getInt(1)
+                chipID = rs.getString(3)
+                print "delete: ", str(objID), chipID
+                Nduplicates += 1
+
+            rs.close()
+        except:
+            print "failed to get duplicates"
+            raise
+
+        print "duplicates found: ", str(Nduplicates)
 
     '''
@@ -935,4 +966,5 @@
 
     '''
+    This function reads the cmf/smf file and loads it into the database as its own table
     Overriding this method. Filter to only import *.psf extensions
     '''