Changeset 40275 for trunk

Timestamp:

Dec 12, 2017, 1:11:39 PM (9 years ago)

Author:

eugene

Message:

add more failure tests and messages; check for PS1_V6 format for the Images.dat (use readFitsHeader method); change MCAL to MCAL_PSF, MCAL_APER

File:

• trunk/ippToPsps/jython/dvo.py (modified) (7 diffs)

trunk/ippToPsps/jython/dvo.py

@@ -101 +101 @@
 
         if not self.loadImages():
+            self.logger.debugPair("FAILED to load images", "exiting")
             os._exit(1)
 
+        print "DONE load images (resetAllTables)"
 
     '''
@@ -145 +147 @@
         self.logger.infoPair("DVO Images.dat file", "NOT up-to-date")
 
+        # check on the format:
+        try: 
+            imagesHeaderPHU = self.readFitsHeader("PHU", path)
+        except Exception,e:
+            # print the error, wait a few secs then try again
+            print str(e)
+            self.logger.infoPair("Failed to find PHU for Images.dat", path)
+            raise
+
+        if "FORMAT" in imagesHeaderPHU:
+            fileFormat = imagesHeaderPHU["FORMAT"]
+        else:
+            self.logger.infoPair("Missing FORMAT for Images.dat", path)
+            raise
+
+        self.logger.infoPair("got FORMAT for Images.dat", fileFormat)
+
+        if not (fileFormat == "PS1_V6"):
+            self.logger.infoPair("Invalid FORMAT for Images.dat (PS1_V6)", fileFormat)
+            return False
+
         self.importFits(
                 path,
-                "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS MCAL SECZ PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM",
+                "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS MCAL_PSF MCAL_APER SECZ PHOTCODE XPIX_SYS_ERR YPIX_SYS_ERR N_FIT_ASTROM MAG_SYS_ERR N_FIT_PHOTOM",
                 self.scratchDb.dvoImagesTable)
         self.logger.infoPair("Adding primary key to", self.scratchDb.dvoImagesTable)
@@ -559 +582 @@
           self.logger.debugPair("Reading IPP table", table.name)
           table = stilts.tpipe(table, cmd='explodeall')
-          #adds an index to all the tables 
-       #   table = stilts.tpipe(table, cmd='addcol table_index $0')
+
+          # adds an index to all the tables 
+          # table = stilts.tpipe(table, cmd='addcol table_index $0')
 
           # IPP FITS files are littered with infinity values. Remove them
@@ -566 +590 @@
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+
+          # verify that all columns in 'columns' exist in the tabl
+          ## myColumns = table.columns()
+          ## for wcol in columns:
+          ##     foundCol = False
+          ##     for tcol in myColumns:
+          ##         if (wcol == tcol.name):
+          ##             print "found wcol " + wcol
+          ##             foundCol = True;
+          ##             break
+          ##     if not foundCol:
 
           attempts = 0
@@ -575 +610 @@
                   table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName)
                   break
-              except:
-                  # wait a few secs then try again
+              except Exception,e:
+                  # print the error, wait a few secs then try again
+                  print str(e)
                   time.sleep(2)
                   attempts = attempts + 1
@@ -663 +699 @@
 
     '''
+    Find and read a header extension.
+    '''
+    def readFitsHeader(self, extname, filename):
+
+        cmd = "fhead -n %s %s" % (extname, filename)
+        p = Popen(cmd, shell=True, stdout=PIPE)
+        output = p.communicate()[0]        
+
+        header = {}
+
+        # split the output header bytes into key/value pairs for each 80-byte line
+        for i in range(0, len(output), 80):
+            # print "line %d : %s" % (i, output[i:i+80])
+            record = output[i:i+80]
+
+            match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-+_\.:\s@#\(\)\,]+)\'*\\/*', record)
+
+            if match:
+                param = match.group(2)
+                value = match.group(3).strip()
+                if value == "NaN": value = "NULL"
+                header[param] = value
+
+        return header
+
+    '''
     ingest skyregion into MySQL database using the native DVO program dvopsps
     Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this
@@ -688 +750 @@
 
         # make sure we have an up-to-date Images table
-        self.loadImages() # this is in nativeIngestDetections (remove?)
+        if not self.loadImages(): # this is in nativeIngestDetections (remove?)
+            self.logger.debugPair("FAILED to load images", "exiting")
+            os._exit(1)
+
+        print "DONE load images (nativeIngestDetections)"
 
         # the box dimensions are the area used to select the items of