Changeset 37551 for trunk/ippToPsps

Timestamp:

Nov 3, 2014, 2:41:20 PM (12 years ago)

Author:

heather

Message:

added diff / ff stages to addstar, partially to ipptopsps

Location:

trunk/ippToPsps

Files:

• config/ippToPspsDbSchema.sql (modified) (1 diff)
• jython/Makefile.am (modified) (1 diff)
• jython/batch.py (modified) (1 diff)
• jython/detectionbatch.py (modified) (1 diff)
• jython/dvoforceddetections.py (added)
• jython/dvoforcedobjects.py (added)
• jython/forcedobjectbatch.py (modified) (1 diff)
• jython/forcedwarpbatch.py (modified) (1 diff)
• jython/gpc1db.py (modified) (2 diffs)
• jython/ipptopspsdb.py (modified) (1 diff)

trunk/ippToPsps/config/ippToPspsDbSchema.sql

@@ -189 +189 @@
 SET character_set_client = @saved_cs_client;
 --
+-- Table structure for table `forcedobject`
+--
+
+DROP TABLE IF EXISTS `forcedobject`;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+CREATE TABLE `forcedobject` (
+  `batch_id` bigint(20) unsigned NOT NULL,
+  `region` varchar(50) default NULL,
+  KEY `batch_id` (`batch_id`),
+  CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+SET character_set_client = @saved_cs_client;
+
+--
 -- Table structure for table `diffobject`
 --

trunk/ippToPsps/jython/Makefile.am

@@ -14 +14 @@
   diffobjectbatch.py \
   dvodetections.py \
+  dvoforceddetections.py \
+  dvoforcedobjects.py \
   dvoobjects.py \
   dvo.py \

trunk/ippToPsps/jython/batch.py

@@ -579 +579 @@
         # for OB, this imports object data from DVO
         if not self.importIppTables():
-            if self.batchType == "OB":
+            if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO"):
                 self.logger.errorPair("skipping this batch", "no dvo tables for this region")
                 self.ippToPspsDb.updateProcessed(self.batchID, 1)

trunk/ippToPsps/jython/detectionbatch.py

@@ -214 +214 @@
         sqlLine.group("completMag",       self.getKeyFloat(header, "%.8f", 'FLIMIT'))
         sqlLine.group("astroScat",        self.getKeyFloat(header, "%.8f", 'CERROR'))
-        sqlLine.group("photoScat",        self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
+        sqlLine.group("photoScat",        self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'))
         sqlLine.group("numAstroRef",      self.getKeyValue(header, 'NASTRO'))
         sqlLine.group("numPhotoRef",      self.getKeyValue(header, 'NASTRO'))

trunk/ippToPsps/jython/forcedobjectbatch.py

@@ +1 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+
+import stilts
+from java.lang import *
+from java.sql import *
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from dvoforcedobjects import DvoForcedObjects # need one specific for cpy ?
+from sqlUtility import sqlUtility
+
+import logging.config
+
+'''
+ForcedObjectBatch class
+
+This class, a sub-class of Batch, processes individual DVO region file pairs (cpt, cpy, and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file.
+
+'''
+class ForcedMeanObjectBatch(Batch):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, 
+                 logger, 
+                 config,
+                 skychunk,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 dvoID,
+                 batchID):
+
+       super(ForcedMeanObjectBatch, self).__init__(
+               logger,
+               config,
+               skychunk,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               dvoID,
+               batchID,
+               "FO", 
+               None)
+
+       try:
+           self.dvoForcedObjects = DvoForcedObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
+       except:
+           self.logger.errorPair("Unable to create instance of", "DvoForcedObjects")
+           raise
+
+       # create an output filename, which is {dvoINDEX}.FITS
+       self.outputFitsFile = "%08d.FITS" % self.id
+       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+
+       # dump stuff to log
+       self.logger.infoPair("DVO INDEX", "%d" % self.id)
+
+    '''
+    Overriden from batch base-class. We import from DVO directly for forced mean objects
+    '''
+    def importIppTables(self, columns="*", filter=""):
+
+        self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id)
+        self.ippToPspsDb.insertForcedMeanObjectMeta(self.batchID, self.region)
+        self.dvoForcedObjects.nativeIngestRegion(self.region)
+
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+
+        if not self.scratchDb.tableExists(cptTableName):
+            self.logger.infoPair("can't find in scratch db:","cpt table")
+            return False
+
+        cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy")
+
+        if not self.scratchDb.tableExists(cpyTableName):
+            self.logger.infoPair("can't find in scratch db:","cpy table")
+            return False
+
+        return True
+
+    '''
+    Applies indexes to the PSPS tables
+    '''
+    def alterPspsTables(self):
+
+        return True
+
+    '''
+    Applies indexes to the IPP tables
+    '''
+    def indexIppTables(self):
+
+        # since dvopsps is now used, no action is needed here
+        # self.logger.infoPair("Creating indexes on", "IPP tables")
+
+        return True
+
+    '''
+    Inserts stuff for all mags
+    '''
+    def updateForcedMeanObjectFromCps(self, cpsTable):
+
+        # list of all filters PSPS is interested in
+        # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
+        
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
+    
+        # get a count of the available filters
+        filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable()
+
+        self.logger.infoPair("Available filters in Photcodes", filters)
+
+        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cps table")
+        for filter in filters:
+
+            filterID = self.scratchDb.getFilterID(filter[1])
+
+            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
+            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
+            # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!)
+
+            # set the MeanObject fields based largely on dvopsps cps fields:
+
+            # XXX EAM 20140724 : filterCount is meant to match
+            # Nsecfilt, but is potentially not determined correctly.
+            # use a call to a dvo-native command which knows how to
+            # find Nsecfilt (or save in the db with dvopsps)
+
+            # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting?
+            # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 
+            #  " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+
+            sql = "UPDATE ForcedMeanObject JOIN \
+                   " + cpsTable + " AS cps ON (cps.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+                   SET \
+                    ForcedMeanObject." + filter[1] + "nTotal         = NWARP \
+                   ,ForcedMeanObject." + filter[1] + "nIncPSFMag     = NUSED_WRP \
+                   ,ForcedMeanObject." + filter[1] + "nIncKronMag    = NUSED_KRON_WRP \
+                   ,ForcedMeanObject." + filter[1] + "nIncApMag      = NUSED_AP_WRP \
+                   ,ForcedMeanObject." + filter[1] + "PSFMag         = MAG_PSF_WRP \
+                   ,ForcedMeanObject." + filter[1] + "PSFMagErr      = MAG_PSF_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "PSFMagStd      = MAG_PSF_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "KronMag        = MAG_KRON_WRP \
+                   ,ForcedMeanObject." + filter[1] + "KronMagErr     = MAG_KRON_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "KronMagStd     = MAG_KRON_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "ApMag          = MAG_AP_WRP \
+                   ,ForcedMeanObject." + filter[1] + "ApMagErr       = MAG_AP_WRP_ERR \
+                   ,ForcedMeanObject." + filter[1] + "ApMagStd       = MAG_AP_WRP_STD \
+                   ,ForcedMeanObject." + filter[1] + "Flags          = FLAGS "
+
+
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed update MeanObject", sql)
+                raise
+
+        # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12)
+        # XXX EAM 20140724 : keep this cut?
+        cut = 0.5
+        self.logger.infoPair("Setting to NULL all MeanMagErr value >", "%f" % cut)
+        for filter in filters:
+
+            sql = "UPDATE ForcedMeanObject \
+                   SET " + filter[1] + "PSFMagErr = null \
+                   WHERE " + filter[1] + "PSFMagErr > " + str(cut)
+            self.scratchDb.execute(sql)
+
+        self.logger.infoPair("Calculating nDetections from", "n[filters]")
+        for filter in filters:
+            # now do a sum of n[filters], but do not include the ones with -999
+            sql  = "UPDATE ObjectThin "
+            sql += "SET nDetections = nDetections + n" + filter[1]
+            sql += " WHERE n" + filter[1] + " != -999"
+            self.scratchDb.execute(sql)
+
+    '''
+    Inserts stuff for all mags from the cpy files
+    '''
+    def updateForcedMeanObjectFromCpy(self, cpyTable):
+
+        # list of all filters PSPS is interested in
+        # XXX EAM : 2014.07.24 : this list should probably be in a config file somewhere
+        interestedFilters = ['g', 'r', 'i', 'z', 'y']
+        
+        filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters)
+    
+        # get a count of the available filters
+        filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable()
+
+        self.logger.infoPair("Available filters in Photcodes", filters)
+
+        # the 'code' now defines the order in the cps file that the mags are listed for a given filter
+        self.logger.infoPair("Adding magnitudes from", "cpy table")
+        for filter in filters:
+
+            filterID = self.scratchDb.getFilterID(filter[1])
+
+            # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000)
+            # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags
+            # XXX EAM : 20140724 this manipulation is no longer needed : [grizy]Flags is now 4 byte (was 8 byte!)
+
+            # set the MeanObject fields based largely on dvopsps cps fields:
+
+            # XXX EAM 20140724 : filterCount is meant to match
+            # Nsecfilt, but is potentially not determined correctly.
+            # use a call to a dvo-native command which knows how to
+            # find Nsecfilt (or save in the db with dvopsps)
+
+            # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting?
+            # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 
+            #  " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+
+            sql = "UPDATE ForcedMeanObject JOIN \
+                   " + cpyTable + " AS cpy ON (cpy.row = (ForcedMeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \
+                   SET \
+                    ForcedMeanObject." + filter[1] + "nIncR5          =  NMEAS \
+                   ,ForcedMeanObject." + filter[1] + "nIncR6          =  NMEAS \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5      =  FLUX_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Err   =  FLUX_ERR_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Std   =  FLUX_STD_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR5Fill  =  FLUX_FIL_AP_R5 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6      =  FLUX_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Err   =  FLUX_ERR_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Std   =  FLUX_STD_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "FmeanflxR6Fill  =  FLUX_FIL_AP_R6 \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX11 =  X11_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX12 =  X12_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearX22 =  X22_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearE1  =  E1_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjSmearE2  =  E2_SM_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX11 =  X11_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX12 =  X12_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearX22 =  X22_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearE1  =  E1_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensObjShearE2  =  E2_SH_OBJ \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX11 =  X11_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX12 =  X12_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearX22 =  X22_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearE1  =  E1_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFSmearE2  =  E2_SM_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX11 =  X11_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX12 =  X12_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearX22 =  X22_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearE1  =  E1_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "LensPSFShearE2  =  E2_SH_PSF \
+                   ,ForcedMeanObject." + filter[1] + "Gamma           =  GAMMA \
+                   ,ForcedMeanObject." + filter[1] + "E1              =  E1 \
+                   ,ForcedMeanObject." + filter[1] + "E2              =  E2 "
+
+
+            try: self.scratchDb.execute(sql)
+            except:
+                self.logger.errorPair("failed update on ForcedMeanObject using cpy table", sql)
+                raise
+
+
+    '''
+    Populates the ForcedMeanObject table
+    '''
+    def populateForcedMeanObjectTable(self):
+
+        cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+        cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps")
+        cpyTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpy")
+
+                
+        self.logger.info("Populating ForcedMeanObject")
+        self.logger.info("Inserting objects from cpt file")
+
+        # note "dec" is a reserved word in MySQL
+        # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems...
+        # INSERT IGNORE INTO ObjectThin
+
+        sqlLine = sqlUtility("INSERT INTO ForcedMeanObject (")
+
+        sqlLine.group("objID",           "EXT_ID") 
+        sqlLine.group("ippObjID",        "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly
+        sqlLine.group("batchID",         "'" + str(self.batchID) + "'")
+        sqlLine.group("nDetections",      "'0'")
+        sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName)
+
+        try:
+            self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("Couldn't populate Object table from cpt", sql)
+            return False
+ 
+        # add row count columns so we can perform joins to get colors
+        self.logger.infoPair("Adding 'row' columns to", "Object and cps tables")
+        self.scratchDb.addRowCountColumn("ForcedMeanObject", "row")
+        self.scratchDb.addRowCountColumn(cpsTableName, "row")
+
+
+        self.logger.infoPair("update ForcedMeanObject from ","cps table")
+
+        self.updateForcedMeanObjectFromCps(cpsTableName)
+
+        self.logger.infoPair("update ForcedMeanObject from ","cpy table")
+
+        self.updateForcedMeanObjectFromCpy(cpyTableName)
+
+
+        self.logger.infoPair("Dropping row column from", "ForcedMeanObject table")
+        self.scratchDb.dropColumn("ForcedMeanObject", "row")
+
+        ##self.logger.infoPair("Purging from scratch Db", self.region + " region")
+
+        ##Don't do this till after MeanObject
+        ##self.dvoForcedObjects.purgeRegion(self.region)
+
+        self.setMinMaxObjID(["ForcedMeanObject"])
+
+        return True
+
+ 
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+
+        if not self.populateForcedMeanObjectTable(): return False
+
+        # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number
+        self.logger.infoPair("Forcing uniqueness on", "objID in ForcedMeanObject table")
+        rowCountBefore = self.scratchDb.getRowCount("ForcedMeanObject")
+
+        # XXX EAM : note that in mysql versions later than 5.1, this fails
+        # unless the following is called first: 
+        # set session old_alter_table=1
+        # follow the command with 
+        # set session old_alter_table=0
+        # OF COURSE, this fails for mysql version < 5.5...
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=1")
+            
+        self.scratchDb.execute("ALTER IGNORE TABLE ForcedMeanObject ADD UNIQUE INDEX(objID)")
+        if self.scratchDb.version > 5.1:
+            self.scratchDb.execute("set session old_alter_table=0")
+
+        rowCountAfter = self.scratchDb.getRowCount("ForcedMeanObject")
+        self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore))
+
+        self.dvoForcedObjects.purgeRegion(self.region)
+
+        #this is abuse of something but this is how I get the object batches to crash to further investigate them
+        
+#        rowCountAfter = self.scratchDb.getRowCount("Object")
+        return True
+#        return False

trunk/ippToPsps/jython/forcedwarpbatch.py

@@ +1 @@
+#!/usr/bin/env jython
+
+import os.path
+import sys
+import glob
+import time
+import stilts
+
+from java.lang import *
+from java.sql import *
+
+from java.lang import Math
+from org.apache.commons.math.special import Erf
+
+from xml.etree.ElementTree import ElementTree, Element, tostring
+
+from batch import Batch
+from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+from sqlUtility import sqlUtility
+
+import logging.config
+
+
+'''
+ForcedWarpBatch class
+
+This class, a sub-class of Batch, processes single exposures in the form of IPP smf files
+
+'''
+class ForcedWarpBatch(Batch):
+
+    '''
+    Constructor
+    '''
+    def __init__(self, 
+                 logger,
+                 config,
+                 skychunk,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 forcedWarpID,
+                 batchID):
+
+       super(ForcedWarpBatch, self).__init__(
+               logger,
+               config,
+               skychunk,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               forcedWarpID,
+               batchID,
+               "P2", 
+               gpc1Db.getForcedWarpStageSmf(forcedWarpID)
+
+       # get camera meta data
+       meta = self.gpc1Db.getForcedWarpStageMeta(self.id)
+       if not meta:
+           self.logger.errorPair("Could not get", "forced warp metadata")
+           raise
+
+       self.expID = meta[0];
+       self.expName = meta[1];
+       self.distGroup = meta[2];
+       self.analysisVer = meta[3];
+       self.bias = meta[4];
+       self.biasScat = meta[5];
+
+       if not self.analysisVer: self.analysisVer = -999
+
+       # create an output filename, which is {expID}.FITS
+       self.outputFitsFile = "%08d.FITS" % self.expID
+       self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
+       self.dropTableVerbose("ForcedWarpMeasurement")
+       self.dropTableVerbose("ForcedWarpExtended")
+       self.dropTableVerbose("ForcedWarpMeta")
+       self.dropTableVerbose("ForcedWarpToImage")
+       if not self.config.retry:
+       self.scratchDb.dropTable("ForcedWarpMeasurement")
+       self.scratchDb.dropTable("ForcedWarpExtended")
+       self.scratchDb.dropTable("ForcedWarpMeta")
+       self.scratchDb.dropTable("ForcedWarpToImage")
+       self.historyModNum = "0"
+
+       # get a few primary header values. if in test mode, then use defaults
+       if self.safeDictionaryAccessWithDefault(self.header, 'MJD-OBS', "1") == "NULL":
+           self.logger.errorPair("Could not get", "MJD-OBS")
+           raise
+
+
+       if self.safeDictionaryAccessWithDefault(self.header, 'EXPTIME', "1") == "NULL":
+           self.logger.errorPair("Could not get", "EXPTIME")
+           raise
+
+       if self.safeDictionaryAccessWithDefault(self.header, 'FILTERID', "g.0000") == "NULL":
+           self.logger.errorPair("Could not get", "FILTERID")
+           raise
+       
+       # MJD-OBS is the exposure start, EXTIME / 172800 = (EXPTIME sec / 84600 sec/day) / 2
+       self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0)
+      
+       # set up some defauts
+       self.calibModNum = 0
+       self.totalNumPhotoRef = 0
+
+
+       self.filter = self.header['FILTERID'][0:1]
+       self.filterID = self.scratchDb.getFilterID(self.filter) 
+
+       # insert what we know about this stack batch into the stack table
+       self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter)
+
+       # dump stuff to log
+       self.logger.infoPair("Cam ID", "%d" % self.id)
+       self.logger.infoPair("Exp ID", "%d" % self.expID)
+       self.logger.infoPair("Exp name", "%s" % self.expName)
+       self.logger.infoPair("Distribution group", "%s" % self.distGroup)
+
+
+
+
+    '''
+    Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header
+    '''
+    def populateForcedWarpMeta(self):
+        self.logger.infoPair("Processing table", "ForcedWarpMeta")
+
+        photoCalID = str(self.scratchDb.getPhotoCalID(forcedWarpID))       
+
+
+        sqlLine = sqlUtility("INSERT INTO ForcedWarpMeta (")
+        #batchID (below)
+        #surveyID (below)
+        #filterdID (below)
+        sqlLine.group("skyCellID",     str(self.skycellID)) 
+        sqlLine.group("photoCalID",    photoCalID) 
+        sqlLine.group("magSat",           self.getKeyFloat(header, "%.8f", 'FSATUR'))
+        sqlLine.group("analVer",           self.analysisVer);
+        sqlLine.group("expTime",               self.getKeyFloat(self.header, "%.8f", 'EXPREQ'));
+        sqlLine.group("completMag",       self.getKeyFloat(header, "%.8f", 'FLIMIT'))
+        sqlLine.group("astroScat",        self.getKeyFloat(header, "%.8f", 'CERROR'))
+        sqlLine.group("photoScat",             self.getKeyFloat(self.header, "%.8f", 'ZPT_ERR'));
+        sqlLine.group("nAstroRef",      self.getKeyValue(header, 'NASTRO'))
+        sqlLine.group("nPhotoRef",      self.getKeyValue(header, 'NASTRO'))
+        sqlLine.group("psfModelID",     self.getKeyValue(header, 'PSFMODEL'))
+        sqlLine.group("psfFwhm_mean",   self.getKeyFloat(header, "%.8f", 'FWHM_MAJ'))
+        sqlLine.group("psfFwhm_max",    self.getKeyFloat(header, "%.8f", 'FW_MJ_UQ'))
+        sqlLine.group("photoZero",        self.getKeyFloat(self.header, "%.8f", 'ZPT_OBS'))
+        #photoColor -- how do I set this? it's also not set in stack meta
+        sqlLine.group("ctype1",                self.getKeyValue(self.header, 'CTYPE1'));
+        sqlLine.group("ctype2",                self.getKeyValue(self.header, 'CTYPE2'));
+        sqlLine.group("crval1",                self.getKeyFloat(self.header, "%.8f", 'CRVAL1'));
+        sqlLine.group("crval2",                self.getKeyFloat(self.header, "%.8f", 'CRVAL2'));
+        sqlLine.group("crpix1",                self.getKeyFloat(self.header, "%.8f", 'CRPIX1'));
+        sqlLine.group("crpix2",                self.getKeyFloat(self.header, "%.8f", 'CRPIX2'));
+        sqlLine.group("cdelt1",                self.getKeyFloat(self.header, "%.8e", 'CDELT1'));
+        sqlLine.group("cdelt2",                self.getKeyFloat(self.header, "%.8e", 'CDELT2'));
+        sqlLine.group("pc001001",              self.getKeyFloat(self.header, "%.8e", 'PC001001'));
+        sqlLine.group("pc001002",              self.getKeyFloat(self.header, "%.8e", 'PC001002'));
+        sqlLine.group("pc002001",              self.getKeyFloat(self.header, "%.8e", 'PC002001'));
+        sqlLine.group("pc002002",              self.getKeyFloat(self.header, "%.8e", 'PC002002'));
+
+
+        sql = sqlLine.make(") VALUES ( ", ")")
+
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql: ', sql)
+            raise
+
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "batchID", str(self.batchID))
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "surveyID", str(self.surveyID))
+        self.scratchDb.updateFilterID("ForcedWarpMeta", self.filter)
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "calibModNum", str(self.calibModNum))
+        self.scratchDb.updateAllRows("ForcedWarpMeta", "dataRelease", str(self.skychunk.dataRelease))
+        self.tablesToExport.append("ForcedWarpMeta")
+       
+    '''
+    Populates the ForcedWarpToImage table
+    '''
+    def populateForcedWarpToImage(self):
+
+        self.logger.infoPair("Procesing table", "ForcedWarpToImage")
+
+        for filter in self.filters:
+            
+            forcedwarpID = self.forcedwarpID
+            if forcedwarpID > 0:
+    
+                imageIDs = self.gpc1Db.getImageIDsForThisForcedWarpID(forcedwarpID)
+
+                for imageID in imageIDs:
+
+                    sql = "INSERT INTO ForcedWarpToImage (forcedwarpID, imageID) \
+                   VALUES (\
+                   " + str(forcedwarpID) + ", " + imageID + ")"
+                    self.scratchDb.execute(sql)
+
+                    # now update StackMeta with correct number of inputs
+
+        self.tablesToExport.append("ForcedWarpToImage")
+
+
+
+
+
+    '''
+    Populates the ForcedWarpMeasurement table
+    '''
+    def populateForcedWarpMeasurementTable(self, results):
+
+        pspsTableName = "ForcedWarpMeasurement"
+        ippTableName = "ForcedWarpMeasurement_psf"
+
+
+        self.logger.infoPair("Procesing table", "ForcedWarpMeasurement")
+
+        # insert the per-object data (IDs, random stack ID, dataRelease, etc)
+        self.selectDvoObjIDs()
+
+        self.generateRandomIDs()
+
+        # add indexes ForcedWarpMeasurement
+        self.scratchDb.createIndex("ForcedWarpMeasurement", "objID")
+
+               
+
+
+
+
+
+
+
+        self.tablesToExport.append("ForcedWarpMeasurement")
+
+
+    '''
+    Populates the ForcedWarpExtended table
+    '''
+    def populateForcedWarpExtendedTable(self, results):
+
+        pspsTableName = "ForcedWarpExtended"
+        ippTableName = "ForcedWarpExtended_psf"
+
+    
+        self.tablesToExport.append("ForcedWarpExtended")
+
+
+    '''
+    Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
+    '''
+    def populatePspsTables(self):
+        self.logger.infoPair("starting","populatePspsTables");
+        self.skipBatch = False
+        # each of the "populate*" methods below add their tables to the list:
+        self.tablesToExport=[]
+    
+        #this should be done
+        self.populateForcedWarpMeta()          
+
+        self.populateForcedWarpMeasurement()           
+        self.populateForcedWarpExtended()
+        self.populateForcedWarpToImage()
+             
+        self.setMinMaxObjID(["ForcedWarpMeasurement"])
+
+        self.logger.infoPair("finishing","populatePspsTables"); 
+        return True
+          
+
+
+    '''
+    This function reads the cmf/smf file and loads it into the database as its own table
+    '''
+    def importIppTables(self, columns="*", tableRE=""):
+        
+        if self.config.retry: return True
+        self.logger.infoPair("Importing FW tables with table match expression: ", tableRE)
+        fileName = self.fits.getPath()
+
+        self.logger.infoPair("using filename:",fileName)
+            
+        try:
+           tables = stilts.treads(fileName)
+        except:
+           self.logger.errorPair("STILTS could not import from", fileName)
+           return False
+        for table in tables:
+              
+           match = re.match(tableRE, table.name)
+           if not match: continue
+           self.logger.infoPair("Reading IPP table", table.name)
+           table = stilts.tpipe(table, cmd='addcol table_index $0')
+           table = stilts.tpipe(table, cmd='explodeall')
+           
+           # drop any previous tables before import
+           self.scratchDb.dropTable(table.name)
+                  
+           # IPP FITS files are littered with infinities, so remove these
+           self.logger.info("Removing Infinity values from all columns")
+           table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+                  
+           try:
+               table.write(self.scratchDb.url + '#' + table.name)
+               count = count + 1
+           except:
+               self.logger.exception("Problem writing table '" + table.name + "' to the database")
+                      
+        self.logger.infoPair("Done. Imported", "%d tables" % count)
+        self.indexIppTables()
+                      
+        return True
+
+
+               
+
+
+
+
+
+
+    def generateRandomIDs(self):
+        sql = "UPDATE ForcedWarpMeasurement set randomWarpID = FLOOR(RAND()*9223372036854775807)";
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair('failed sql',sql)
+            return
+
+
+
+
+
+
+
+
+
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("(ForcedWarp.*)")

trunk/ippToPsps/jython/gpc1db.py

@@ -116 +116 @@
             
             stage = "forcedwarp"
-            sql = "SELECT differentStuff from Stufftable"
+            sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM fullForceInput \
+                   JOIN addRun ON(fullForceInput.ff_id = addRun.stage_id, fullForceInput.warp_id = addRun.stage_extra1) \
+                   JOIN minidvodbRun USING(minidvodb_name) \
+                   JOIN minidvodbProcessed USING(minidvodb_id) \
+                   JOIN mergedvodbRun USING(minidvodb_id) \
+                   JOIN mergedvodbProcessed USING (merge_id) \
+                   JOIN fullForceRun USING(ff_id) \
+                   JOIN skycalRun using (skycal_id)
+                   JOIN stackRun USING(stack_id) \
+                   JOIN stackSumSkyfile using (stack_id) \
+                   JOIN skycell USING(skycell_id) \
+                   WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \
+                   AND minidvodbRun.state = 'merged' \
+                   AND minidvodbProcessed.fault = 0 \
+                   AND mergedvodbRun.state = 'full' \
+                   AND mergedvodbProcessed.fault = 0 \
+                   AND addRun.stage = '" + stage + "' \
+                   AND addRun.state = 'full' \
+                   AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
+                   AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
+                   AND mjd_obs <= (to_days('" + tend + "') - 678941) "
 
         try:
@@ -165 +186 @@
 
         return imageIDs
+
+    
+
+
+
 
     '''

trunk/ippToPsps/jython/ipptopspsdb.py

@@ -756 +756 @@
 
         self.execute(sql)
+
+    '''
+    Inserts some forced detection metadata for this batch ID
+    '''
+    def insertForcedWarpMeta(self, batchID, expID, filter):
+
+        sql = "INSERT INTO forcedwarp ( \
+               batch_id \
+               ,exp_id \
+               ,filter \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ," + str(expID) + " \
+               ,'" + filter + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+    '''
+    Inserts some forcedobject metadata for this batch ID
+    '''
+    def insertForcedObjectMeta(self, batchID, region):
+
+        sql = "INSERT INTO forcedobject ( \
+               batch_id \
+               ,region \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + region + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+
+    ''' 
+    Inserts some diff metadata for this batch ID
+    '''
+    def insertDiffMeta(self, batchID, filter, diffType):
+
+        sql = "INSERT INTO stack ( \
+               batch_id \
+               ,filter \
+               ,diff_type \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + filter + "' \
+               ,'" + diffType + "')"
+
+        self.execute(sql)
+
+
+
+
+    '''
+    Inserts some diffobject metadata for this batch ID
+    '''
+    def insertDiffObjectMeta(self, batchID, region):
+
+        sql = "INSERT INTO diffobject ( \
+               batch_id \
+               ,region \
+               ) VALUES ( \
+               " + str(batchID) + " \
+               ,'" + region + "' \
+               )"
+
+        self.execute(sql)
+
+
+
+
+
+
     '''
     Resets a batch ready for re-loading to the datastore