Changeset 35097 for trunk/ippToPsps/jython/batch.py

Timestamp:

Feb 6, 2013, 3:16:35 PM (13 years ago)

Author:

eugene

Message:

upgrade to ippToPsps (see doc/upgrade.txt): adds native dvo to mysql ingest operations, adds autogen configuration & installation, splits out global config information from new "skychunk" information (current region on the sky being processed), adds test suites

Location:

trunk/ippToPsps

Files:

• . (modified) (1 prop)
• jython (modified) (1 prop)
• jython/batch.py (modified) (16 diffs)

trunk/ippToPsps/jython

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in

trunk/ippToPsps/jython/batch.py

@@ -30 +30 @@
                  logger, 
                  config,
+                 skychunk,
                  gpc1Db,
                  ippToPspsDb,
@@ -39 +40 @@
                  useFullTables): 
 
+        # print "starting the batch"
+
+        # self.printline = 0
+        # self.testprint()
+
         self.readHeader = False
         self.config = config
+        self.skychunk = skychunk
         self.fits = fits
         self.useFullTables = useFullTables
@@ -48 +55 @@
         self.logger.infoSeparator()
         self.logger.debug("Batch class constructor")
+
+        # self.testprint()
 
         # set up class variables
@@ -56 +65 @@
         self.scratchDb = scratchDb
         self.batchType = batchType;
-        self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
+        self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot"
+
+        # self.testprint()
 
         # check FITS file is ok
@@ -64 +75 @@
             # now check that the fits header is readable
             if not self.header: 
-                logger.errorPair("Could not read FITS for id", "%d" % id)
+                logger.errorPair("Could not read FITS PHU for id", "%d" % id)
                 raise
 
@@ -72 +83 @@
         self.tablesToExport = []
 
-        if self.config.survey != "":
-            self.surveyID = self.scratchDb.getSurveyID(self.config.survey)
+        if self.skychunk.survey != "":
+            self.surveyID = self.scratchDb.getSurveyID(self.skychunk.survey)
         else:
             self.surveyID = -1;
        
         # create datastore object
-        self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb)
-
-        # get local storage location from config
+        self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
+
+        # get local storage location from skychunk
         self.batchName = Batch.getNameFromID(self.batchID)
         self.subDir = Batch.getSubDir(
-                self.config.basePath,
+                self.skychunk.basePath,
                 self.batchType, 
-                self.config.dvoLabel)
+                self.skychunk.dvoLabel)
 
         self.localOutPath = Batch.getOutputPath(
-                self.config.basePath,
+                self.skychunk.basePath,
                 self.batchType, 
-                self.config.dvoLabel,
+                self.skychunk.dvoLabel,
                 self.batchID)
 
         if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
+
+        # self.testprint()
 
         # store today's date
@@ -112 +125 @@
             self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount())
 
+        # self.testprint()
+
         self.logger.infoPair("Output path", self.localOutPath)
 
     
+    def testprint(self):
+      print "here ", self.printline
+      self.printline += 1
+
     '''
     Static method to generated batch name from batch ID
@@ -232 +251 @@
         root.attrib['type'] = self.batchType
         root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") 
-        root.attrib['survey'] = self.config.pspsSurvey
+        root.attrib['survey'] = self.skychunk.pspsSurvey
 
         # min/max object IDs
@@ -273 +292 @@
         # set up filenams and paths
         tarFile = Batch.getTarFile(self.batchID)
-        tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
+        tarPath = Batch.getTarPath(self.skychunk.basePath, self.batchType, self.skychunk.dvoLabel, self.batchID)
         tarballFile = Batch.getTarballFile(self.batchID)
-        tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID)
+        tarballPath = Batch.getTarballPath(self.skychunk.basePath, self.batchType, self.skychunk.dvoLabel, self.batchID)
 
         # tar directory
@@ -376 +395 @@
       self.logger.infoPair("Importing tables with filter", filter)
 
+      # print "trying to read ", self.fits.getPath()
+
       try:
           tables = stilts.treads(self.fits.getPath())
@@ -385 +406 @@
       for table in tables:
 
+          # print "import smf table ", table
           match = re.match(filter, table.name)
           if not match: continue
           self.logger.debugPair("Reading IPP table", table.name)
           table = stilts.tpipe(table, cmd='explodeall')
+
+          # print "read smf table ", table
 
           # drop any previous tables before import
@@ -398 +422 @@
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+          # print "cleaned up values ", table
 
           try:
               table.write(self.scratchDb.url + '#' + table.name)
+              # print "wrote to mysql ", table
               self.scratchDb.killLastConnectionID()
               count = count + 1
@@ -488 +514 @@
 
         # TODO path to DVO prog hardcoded temporarily
-        cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.config.dvoLocation
+        cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.skychunk.dvoLocation
         self.logger.infoPair("Running DVO", cmd)
         p = Popen(cmd, shell=True, stdout=PIPE)
@@ -504 +530 @@
     '''
     Creates and publishes a batch
-    TODO all methods call below should throw exceptions on failure
+    TODO all method calls below should throw exceptions on failure
     '''
     def run(self):
 
+        # this is badly named : it creates the tables and then fills in
+        # the ImageMeta tables for each ota (alterPspsTables)
         if not self.createEmptyPspsTables():
             self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables")
@@ -513 +541 @@
             raise
 
+        # for P2/ST, this reads the detection tables from the CMF/SMF file(s)
+        # for OB, this imports object data from DVO
         if not self.importIppTables():
             self.logger.errorPair("Aborting this batch", "could not import IPP tables")
@@ -529 +559 @@
    
         if self.writeBatchManifest():
-            if self.config.datastorePublishing: 
+            if self.skychunk.datastorePublishing: 
 
                 # tar and zip ready for publication to datastore

@@ -1 +1 @@
 *.class
+Makefile
+Makefile.in