Changeset 32819

Timestamp:

Nov 25, 2011, 11:17:01 AM (15 years ago)

Author:

rhenders

Message:

improved logic so that batches are only marked as 'successfully' processed once they are tar'd and zip'd

File:

• trunk/ippToPsps/jython/batch.py (modified) (7 diffs)

trunk/ippToPsps/jython/batch.py

@@ -286 +286 @@
     Publishes this batch to the datastore
     '''
-    def tarZipAndpublishToDatastore(self):
+    def tarAndZip(self):
       
         # set up filenams and paths
@@ -295 +295 @@
 
         # tar directory
-        p = Popen("tar -cvf " + tarPath + "\
-                -C " + self.subDir + " \
-                " + self.batchName, shell=True, stdout=PIPE)
+        cmd = "tar -cvf " + tarPath + " -C " + self.subDir + " " + self.batchName
+        self.logger.infoPair("Running", cmd)
+        p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
 
+        if p.returncode != 0:
+            self.logger.errorPair("tar command", "failed")
+            return False
+
         # zip tar archive
-        p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE)
+        cmd = "gzip -c " + tarPath + " > " + tarballPath
+        self.logger.infoPair("Running", cmd)
+        p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
+
+        if p.returncode != 0:
+            self.logger.errorPair("gzip command", "failed")
+            return False
+
+        # only now can we report that the batch has successfully processed
+        self.logger.infoPair("Batch tar 'n zip successful", "setting batch to 'processed'")
+        self.ippToPspsDb.updateProcessed(self.batchID, 1)
 
         # delete tar file and original directory
@@ -308 +322 @@
         shutil.rmtree(self.localOutPath)
 
-        # now publish to the datastore
-        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
+        return True
 
     '''
@@ -448 +461 @@
         try:
             stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
-            self.ippToPspsDb.updateProcessed(self.batchID, 1)
         except:
             self.logger.exception("Could not write to FITS")
@@ -494 +506 @@
         p = Popen(cmd, shell=True, stdout=PIPE)
         p.wait()
-        #        out = p.stdout.read()
 
         rowCount = self.scratchDb.getRowCount("dvoDetection")
@@ -507 +518 @@
     '''
     Creates and publishes a batch
-    TODO all methdds call below should throw exceptions on failure
+    TODO all methods call below should throw exceptions on failure
     '''
     def run(self):
@@ -530 +541 @@
             else:
                 self.writeBatchManifest()
-                if int(self.doc.find("options/publishToDatastore").text): self.tarZipAndpublishToDatastore()
+                if int(self.doc.find("options/publishToDatastore").text): 
+                    # tar and zip ready for publication to datastore
+                    if self.tarAndZip():
+                        # now publish to the datastore
+                        tarballFile = Batch.getTarballFile(self.batchID)
+                        Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile)
+
                 if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False)