Changeset 35097 for trunk/ippToPsps/jython/dvo.py

Timestamp:

Feb 6, 2013, 3:16:35 PM (13 years ago)

Author:

eugene

Message:

upgrade to ippToPsps (see doc/upgrade.txt): adds native dvo to mysql ingest operations, adds autogen configuration & installation, splits out global config information from new "skychunk" information (current region on the sky being processed), adds test suites

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython (modified) (1 prop)
• jython/dvo.py (modified) (12 diffs)

trunk/ippToPsps/jython

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

trunk/ippToPsps/jython/dvo.py

@@ -33 +33 @@
 
     '''
-    def __init__(self, logger, config, scratchDbName="ipptopsps_scratch"):
+    def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
 
         # set up logging
         self.logger = logger
         self.config = config
+        self.skychunk = skychunk
+        self.ippToPspsDb = ippToPspsDb
 
         # set up empty lists
@@ -46 +48 @@
         self.regionsIngestedButOutOfDate = []
        
-        # connect to the specified scratch database
+        # connect to the specified scratch database, if a name is given, otherwise use the first version
         try:
-            self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName)
+            self.scratchDb = ScratchDb(self.logger, self.config, '1', scratchDbName)
         except: raise
 
+        # we have (still) 3 modes of ingest from dvo cpt/cps/cpm files 
+        # 1) dvograbber (useFullTables == FALSE), 
+        # 2) jython/stilts (useStilts == TRUE, useFullTables == TRUE)
+        # 3) dvopsps (useStilts == FALSE, useFullTables == TRUE)
+        # my goal (EAM, 2013/01/31) is to migrate to only dvopsps
+
+        self.useStilts = 0
         self.scratchDb.setUseFullTables(True)
 
         # decide if we are using the right DVO for this scratchDb
-        self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation)
+        self.correctDvo = self.scratchDb.isCorrectDvo(self.skychunk.dvoLocation)
 
         # this scratch Db either has no DVO info ingested, or a different DVO ingested. Prompt user
         if not self.correctDvo:
             print "*******************************************************************************"
-            self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "'?")
+            self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.skychunk.dvoLocation + "'?")
             response = raw_input("(y/n) ")
             if response == "y":
@@ -101 +110 @@
 
         # check if we have up-to-date version
-        path = self.config.dvoLocation + "/Photcodes.dat"
+        path = self.skychunk.dvoLocation + "/Photcodes.dat"
         if self.scratchDb.alreadyImportedThisDvoTable(path): 
             self.logger.debugPair("DVO Photcodes.dat file", "up-to-date")
@@ -126 +135 @@
 
         # check if we have up-to-date version
-        path = self.config.dvoLocation + "/Images.dat"
+        path = self.skychunk.dvoLocation + "/Images.dat"
         if self.scratchDb.alreadyImportedThisDvoTable(path): 
             self.logger.debugPair("DVO Images.dat file", "up-to-date")
@@ -151 +160 @@
         if not self.correctDvo: return False
 
-        path =  self.config.dvoLocation + "/SkyTable.fits"
+        path =  self.skychunk.dvoLocation + "/SkyTable.fits"
         if self.scratchDb.alreadyImportedThisDvoTable(path): 
             self.logger.debugPair("DVO SkyTable.fits file", "up-to-date")        
@@ -158 +167 @@
         self.logger.infoSeparator()
         self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date")        
-        self.importFits(
+
+        if self.useStilts:
+            self.importFits(
                 path,
                 "R_MIN R_MAX D_MIN D_MAX INDEX NAME",
                 self.scratchDb.dvoSkyTable)
+        else:
+            
+            # create dvoSkyTable 
+            sql = "drop TABLE dvoSkyTable"
+            try:
+                self.scratchDb.execute(sql)
+            except: pass
+
+            # create dvoSkyTable 
+            sql = "CREATE TABLE dvoSkyTable (R_MIN REAL, R_MAX REAL, D_MIN REAL, D_MAX REAL, INDEX_ INT, NAME CHAR(18))"
+            self.scratchDb.execute(sql)
+
+            # TODO path to DVO prog hardcoded temporarily
+            cmd = "dvopsps skytable"
+            cmd += " -dbhost " + self.scratchDb.dbHost
+            cmd += " -dbname " + self.scratchDb.dbName
+            cmd += " -dbuser " + self.scratchDb.dbUser
+            cmd += " -dbpass " + self.scratchDb.dbPass
+            cmd += " -D CATDIR " + self.skychunk.dvoLocation
+
+            self.logger.infoPair("Running dvopsps", cmd)
+            p = Popen(cmd, shell=True, stdout=PIPE)
+            p.wait()
 
         self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable)
@@ -168 +202 @@
         
         self.scratchDb.setImportedThisDvoTable(path)
-        self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation)
+        self.logger.infoPair("Finished importing SkyTable at", self.skychunk.dvoLocation)
 
         return True
@@ -231 +265 @@
            paths = []
            for fileType in self.ingestFileTypes:
-               paths.append(self.config.dvoLocation + "/" + regionPath + "." + fileType)
+               paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
 
            # check for the existence of all interested file types
@@ -304 +338 @@
         # go no further if we've already partly ingested a different DVO
         if not self.correctDvo:
-            self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation)
+            self.logger.infoPair("Wrong DVO in use", self.skychunk.dvoLocation)
             return
 
@@ -387 +421 @@
         size = 0.0
         for region in regions:
-
+             self.logger.infoPair("sizes for region", region)
              # get combined size of all interested files
              for fileType in self.ingestFileTypes:
-                 size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + "." + fileType)
+                 size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType)
+                 # EAM TEST I/O
+                 self.logger.infoPair(fileType, size)
 
         return size
@@ -519 +555 @@
           try:
               if attempts > 0: self.logger.infoPair("Attempt %d to write" % attempts, table.name)
-              table.cmd_keepcols("CODE NAME TYPE_AS_INT").write(self.scratchDb.url + '#' + tableName)
+              table.cmd_keepcols("CODE NAME TYPE_AS_INT C_LAM K").write(self.scratchDb.url + '#' + tableName)
               break
           except:
@@ -532 +568 @@
       return tableName
 
+
+    '''
+    ingest skyregion into MySQL database using the native DVO program dvopsps
+    Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this
+    includes purging detections outside sky area
+    '''
+    def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize):
+
+        # XXX put the chunk below in a separate method 
+        # blow away existing dvoDetection table
+
+        # clear the 'ingested' field for all boxes
+        self.ippToPspsDb.clearIngestedBoxes()
+        
+        # drop detections table
+        self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
+        self.scratchDb.dropTable(self.scratchDb.dvoDetectionTable)
+        
+        # create detections table
+        self.logger.infoPair("Creating table", self.scratchDb.dvoDetectionTable)
+        sql = "CREATE TABLE " + self.scratchDb.dvoDetectionTable + " LIKE dvoDetection"
+        try: self.scratchDb.execute(sql)
+        except:
+            self.logger.errorPair("Unable to create table", self.scratchDb.dvoDetectionTable)
+            return False
+        self.scratchDb.changeEngineToInnoDB(self.scratchDb.dvoDetectionTable)
+        
+        # add fileID column
+        sql = "ALTER TABLE " + self.scratchDb.dvoDetectionTable + " ADD fileID INT NOT NULL"
+        try: self.scratchDb.execute(sql)
+        except: 
+            self.logger.errorPair("Unable to add fileID column to ", self.scratchDb.dvoDetectionTable)
+            return False
+
+        # add an index to the fileID column
+        self.scratchDb.createIndex(self.scratchDb.dvoDetectionTable, "fileID")
+
+        # make sure we have an up-to-date Images table
+        self.loadImages()
+
+        # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
+        halfSize = boxSize / 2.0
+        # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize
+
+        # TODO path to DVO prog hardcoded temporarily
+        cmd = "dvopsps detections"
+        cmd += " -dbhost " + self.scratchDb.dbHost
+        cmd += " -dbname " + self.scratchDb.dbName
+        cmd += " -dbuser " + self.scratchDb.dbUser
+        cmd += " -dbpass " + self.scratchDb.dbPass
+        cmd += " -D CATDIR " + self.skychunk.dvoLocation
+        cmd += " -region "
+        cmd += " " + str(raCenter-halfSize)
+        cmd += " " + str(raCenter+halfSize)
+        cmd += " " + str(decCenter-halfSize)
+        cmd += " " + str(decCenter+halfSize)
+
+        if self.skychunk.parallel:
+            cmd += " -parallel"
+
+        self.logger.infoPair("Running dvopsps", cmd)
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        p.wait()
+
+        self.ippToPspsDb.setIngestedBox(boxId)
+
+        # add fileID column
+        sql = "ANALYZE TABLE " + self.scratchDb.dvoDetectionTable
+        try: self.scratchDb.execute(sql)
+        except: 
+            self.logger.errorPair("Unable to analyze mysql table", self.scratchDb.dvoDetectionTable)
+            return False
+
+        # update lists after attempted sync
+        # self.printSummary()
+
+        return True

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in