Changeset 35016

Timestamp:

Jan 22, 2013, 9:43:22 AM (13 years ago)

Author:

eugene

Message:

add dvopsps-based method to ingest detections; do not use foreign key constraint if using dvopsps

Location:

branches/eam_branches/ipp-20121219/ippToPsps/jython

Files:

• dvo.py (modified) (1 diff)
• dvodetections.py (modified) (1 diff)
• loader.py (modified) (1 diff)
• scratchdb.py (modified) (1 diff)

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvo.py

@@ -562 +562 @@
       return tableName
 
+
+    '''
+    ingest skyregion into MySQL database using the native DVO program dvopsps
+    Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this
+    includes purging detections outside sky area
+    '''
+    def nativeIngestDetections(self, raCenter, decCenter, boxSize):
+
+        # self.scratchDb.resetAllDvoTables()
+
+        # make sure we have an up-to-date Images table
+        self.loadImages()
+
+        # XXX blow away existing dvoDetection table?
+
+        # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
+        halfSize = boxSize / 2.0
+        # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize
+
+        # TODO path to DVO prog hardcoded temporarily
+        cmd = "dvopsps detections"
+        cmd += " -dbhost " + self.scratchDb.dbHost
+        cmd += " -dbname " + self.scratchDb.dbName
+        cmd += " -dbuser " + self.scratchDb.dbUser
+        cmd += " -dbpass " + self.scratchDb.dbPass
+        cmd += " -D CATDIR " + self.config.dvoLocation
+        cmd += " -region "
+        cmd += " " + str(raCenter-halfSize)
+        cmd += " " + str(raCenter+halfSize)
+        cmd += " " + str(decCenter-halfSize)
+        cmd += " " + str(decCenter+halfSize)
+
+        self.logger.infoPair("Running dvopsps", cmd)
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        p.wait()
+
+
+        # update lists after attempted sync
+        # self.printSummary()
+
+        return True

branches/eam_branches/ipp-20121219/ippToPsps/jython/dvodetections.py

@@ -166 +166 @@
 
        return True
-
-

branches/eam_branches/ipp-20121219/ippToPsps/jython/loader.py

@@ -160 +160 @@
                     useFullTables = 0
                     if batchType != "OB":
-                        # look in DVO for this box (with extra border)
-                        self.logger.infoPair("Querying DVO for this sky area", "")
-                        self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
-                        #self.dvoDetections.setSkyArea()
-                        sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
-                        if sizeToBeIngested == 0.0: smfsPerGB = 999999999
-                        else: smfsPerGB = len(ids)/sizeToBeIngested
-                        self.logger.infoPair("DVO to be ingested", "%7.1e GB" % sizeToBeIngested)
-                        self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
-
-                        # should do we pre-ingest stuff from DVO?
-                        # NOTE EAM : this chunk loads the dvo detections into the mysql db
-                        # XXXX EAM : this should happen for both P2 and Stack detection
-                        # XXXX EAM : in parallel model, this should happen for all P2 & Stack batches
-                        # 
-                        if (batchType == 'P2' or batchType == 'ST') and smfsPerGB > 30:
-                            if not self.dvoDetections.sync():
-                                self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
-                                continue
-     
+
+                        if (self.dvoDetections.useStilts):
+                            # look in DVO for this box (with extra border)
+                            self.logger.infoPair("Querying DVO for this sky area", "")
+                            self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
+                            #self.dvoDetections.setSkyArea()
+                            sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
+                            if sizeToBeIngested == 0.0: smfsPerGB = 999999999
+                            else: smfsPerGB = len(ids)/sizeToBeIngested
+                            self.logger.infoPair("DVO to be ingested", "%7.1e GB" % sizeToBeIngested)
+                            self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB)
+
+                            # should do we pre-ingest stuff from DVO?
+                            # NOTE EAM : this chunk loads the dvo detections into the mysql db
+                            # XXXX EAM : this should happen for both P2 and Stack detection
+                            # XXXX EAM : in parallel model, this should happen for all P2 & Stack batches
+                            # 
+                            if (batchType == 'P2' or batchType == 'ST') and smfsPerGB > 30:
+                                if not self.dvoDetections.sync():
+                                    self.logger.errorPair("Could not sync DVO with MySQL", "skipping")
+                                    continue
+                                
+                                useFullTables = 1
+                        else:
+                            # if we are using the native loader, always use it
+                            self.dvoDetections.nativeIngestDetections(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
                             useFullTables = 1
                 

branches/eam_branches/ipp-20121219/ippToPsps/jython/scratchdb.py

@@ -485 +485 @@
        self.createIndex(self.dvoDetectionTable, "fileID")
 
-       # now add a delete cascading foreign key constraint on fileID
-       sql = "ALTER TABLE dvoDetectionFull \
+       # XXX : this should be based on choice of useStilts (which is set in dvo.py and must be moved)
+       if (False):
+           # now add a delete cascading foreign key constraint on fileID
+           sql = "ALTER TABLE dvoDetectionFull \
               ADD CONSTRAINT fk_fileID \
               FOREIGN KEY (fileID) \
               REFERENCES dvoDone(id) \
               ON DELETE CASCADE"
-       try: self.execute(sql)
-       except: 
-           self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable)
-           return False
+           try: self.execute(sql)
+           except: 
+               self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable)
+               return False
 
        return True