Changeset 36072 for branches/eam_branches/ipp-20130711/ippToPsps

Timestamp:

Aug 30, 2013, 4:52:14 PM (13 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20130711/ippToPsps

Files:

• . (modified) (1 prop)
• config/changes.txt (copied) (copied from trunk/ippToPsps/config/changes.txt )
• config/ippToPspsDbSchema.sql (modified) (1 diff)
• jython/Makefile.am (modified) (1 diff)
• jython/detectionbatch.py (modified) (1 diff)
• jython/gpc1db.py (modified) (2 diffs)
• jython/ipptopspsdb.py (modified) (2 diffs)
• jython/queue.py (modified) (1 diff)
• jython/removepending.py (copied) (copied from trunk/ippToPsps/jython/removepending.py )

branches/eam_branches/ipp-20130711/ippToPsps/config/ippToPspsDbSchema.sql

@@ -127 +127 @@
   `active` tinyint(1) default '1',
   `parallel` tinyint(1) default '0',
+  `P2_smf_version` varchar(16) default NULL,
+  `ST_cmf_version` varchar(16) default NULL, 
   UNIQUE KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

branches/eam_branches/ipp-20130711/ippToPsps/jython/Makefile.am

@@ -30 +30 @@
   pslogger.py \
   queue.py \
+  removepending.py \ 
   scratchdb.py \
   setupScratchDb.py \

branches/eam_branches/ipp-20130711/ippToPsps/jython/detectionbatch.py

@@ -54 +54 @@
                batchID,
                "P2", 
-               gpc1Db.getCameraStageSmf(camID),
+               gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version),
                useFullTables)
 

branches/eam_branches/ipp-20130711/ippToPsps/jython/gpc1db.py

@@ -212 +212 @@
     Gets a camera-stage smf for this cam_id
     '''
-    def getCameraStageSmf(self, camID):
-
+    def getCameraStageSmf(self, camID, smfversion):
+        self.logger.infoPair("using", smfversion)
         self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID))
 
@@ -237 +237 @@
         # list all smf files if a neb path
         files = []
-        if path.startswith("neb"):
-
-            # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
-            f=os.popen("neb-ls -p "+path+"%smf")
-            for i in f.readlines():
-                files.append(i.rstrip())
-
-        # or not a neb path
-        else:
-            files = glob.glob(path + ".smf")
+
+
+        #there are a couple of states for smfversion
+
+        # "use_new" => we have reprocessed the data, use the .smf file
+
+        # "not_reproc" => this has not been reprocessed, use the .smf file
+
+        # "use_original" => we *may* have reprocessed the data, 
+        #    use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig
+        #    use (file).smf if camRun.state = 'full' and if there is only a .smf
+        #    fault out if camRun.state != 'full' -- that is a race condition we don't want.
+       
+        
+        if (smfversion == "not_reproc") or (smfversion == "use_new"):
+            if path.startswith("neb"):
+
+                # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                f=os.popen("neb-ls -p "+path+"%smf")
+                for i in f.readlines():
+                    files.append(i.rstrip())
+
+                # or not a neb path
+            else:
+                files = glob.glob(path + ".smf")
+
+        if (smfversion == "use_original"):
+            # get state of camRun
+            sql = "SELECT state \
+               FROM camProcessedExp \
+               JOIN camRun USING(cam_id) \
+               WHERE camRun.cam_id = %d" % camID
+            try:
+                rs = self.executeQuery(sql)
+                rs.first()
+            except:
+                self.logger.errorPair("Can't query for camRun.state for cam_id",  str(camID))
+                return None
+
+            try:
+                # get state of camRun.cam_id
+                camrunstate = rs.getString(1)
+            except:
+                self.logger.errorPair("No state found for cam_id", str(camID) )
+                return None
+            
+            if (camrunstate == "full"):
+              self.logger.infoPair("camRun.state is ",camrunstate)   
+                 # if full find smf files
+              if path.startswith("neb"):
+
+                 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right?
+                 fsmf=os.popen("neb-ls -p "+path+"%smf")
+                 fsmfs = fsmf.readlines()
+                 fsmforig=os.popen("neb-ls -p "+path+"%smf.original")
+                 fsmforigs = fsmforig.readlines()
+                 smfCount = len(fsmfs)
+                 smfOrigCount = len (fsmforigs)
+
+                 if (smfCount > 0  and smfOrigCount > 0):
+                     #smf and original  so use original
+                     self.logger.infoPair("using:", path+"%smf.original")
+                     for i in fsmforigs:
+                         files.append(i.rstrip())
+
+                 if (smfCount > 0 and smfOrigCount == 0):
+                     self.logger.infoPair("using:", path+"%smf")
+                     #smf file and no smf.original use .smf
+                     for i in fsmfs:
+                         files.append(i.rstrip())
+
+                 #if not smf dont' find files ??)        
+                 # or not a neb path
+
+              else:
+                # this is broken for non neb - on the other hand we probably won't do .original if so
+                files = glob.glob(path + ".smf")
+
+
+
+            # if .smf and .original use .original
+            else:
+                self.logger.infoPair("camRun.state not full, can't ingest", str(camID) )
+                return None
+
 
         # XXX EAM : test output

branches/eam_branches/ipp-20130711/ippToPsps/jython/ipptopspsdb.py

@@ -1109 +1109 @@
         ,queue_OB \
         ,parallel \
+        ,P2_smf_version \
+        ,ST_cmf_version \
         FROM skychunk \
         WHERE name = '" + self.skychunk.name + "'"
@@ -1155 +1157 @@
 
             if rs.getInt(22) == 1: self.skychunk.parallel = True
+            self.skychunk.P2_smf_version = rs.getString(23)
+            self.skychunk.ST_cmf_version = rs.getString(24)
 
             if self.skychunk.parallel: print "USING parallel"

branches/eam_branches/ipp-20130711/ippToPsps/jython/queue.py

@@ -71 +71 @@
             for batchType in self.skychunk.batchTypes: 
                 self.logger.infoPair("BatchType",batchType)
-                # self.logger.infoTitle("Previous failed datastore loads")
-                # self.publishAnyUnpublishedBatches(batchType)
+                #self.logger.infoTitle("Previous failed datastore loads")
+                #self.publishAnyUnpublishedBatches(batchType)
 
                 # get totals for whole area to check if there is anything to do