- Timestamp:
- Jan 12, 2015, 12:24:24 PM (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-pv3-20140717-merge/ippToPsps/jython/batch.py
r36726 r37820 37 37 batchID, 38 38 batchType, 39 fits, 40 useFullTables): 41 42 # print "starting the batch" 43 44 # self.printline = 0 45 # self.testprint() 39 fits): 46 40 47 41 self.readHeader = False … … 49 43 self.skychunk = skychunk 50 44 self.fits = fits 51 self.useFullTables = useFullTables52 45 53 46 # set up logging … … 55 48 self.logger.infoSeparator() 56 49 self.logger.debug("Batch class constructor") 57 58 # self.testprint()59 50 60 51 # set up class variables … … 68 59 self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot" 69 60 70 # self.testprint()71 72 61 # check FITS file is ok 73 62 if self.fits: … … 79 68 raise 80 69 81 self.scratchDb.setUseFullTables(self.useFullTables) 70 # define the dvo table names 71 self.scratchDb.setDvoTableNames() 82 72 83 73 # TODO … … 107 97 if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) 108 98 109 # self.testprint()110 111 99 # store today's date 112 100 now = datetime.datetime.now(); 113 101 self.dateStr = now.strftime("%Y-%m-%d") 114 102 115 # create DVO tables if accessing DVO directly116 if not self.useFullTables:self.scratchDb.createDvoTables()103 # create DVO tables (scratch database) 104 self.scratchDb.createDvoTables() 117 105 118 106 # dump stuff to the log … … 120 108 self.logger.infoPair("Batch name", self.batchName) 121 109 self.logger.infoPair("Survey ID", "%d" % self.surveyID) 122 self.logger.infoBool("Use full DVO tables?", self.useFullTables)123 110 124 111 if self.fits: 125 112 self.logger.infoPair("Input FITS file", self.fits.getOriginalPath()) 126 113 self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount()) 127 128 # self.testprint()129 114 130 115 self.logger.infoPair("Output path", self.localOutPath) … … 237 222 self.logger.errorPair("Missing header field", key) 238 223 return "NULL" 224 225 ''' 226 Returns the string keyword value from this header or else "NULL" 227 ''' 228 def getKeyValue(self, header, key): 229 230 if key in header: return header[key] 231 else: 232 self.logger.errorPair("Missing header field", key) 233 return "NULL" 234 235 ''' 236 Returns the string keyword value from this header or else "NULL" 237 ''' 238 def getKeyFloat(self, header, format, key): 239 240 if key in header: 241 value = format % float(header[key]) 242 else: 243 self.logger.errorPair("Missing header field", key) 244 value = format % -999.9 245 246 return value 247 248 ''' 249 Returns the string keyword value from this header or else "NULL" 250 ''' 251 def getKeyInt(self, header, defValue, key): 252 253 if key in header: 254 value = str(int(header[key])) 255 else: 256 self.logger.errorPair("Missing header field", key) 257 value = str(defValue) 258 259 return value 239 260 240 261 ''' … … 362 383 self.logger.infoPair("Max objID", "%ld" % self.maxObjID) 363 384 385 386 ''' 387 Sets min and max obj ID using the provided table, or list of tables 388 ''' 389 def setMinMaxDiffObjID(self, tables): 390 first = True 391 self.totalDetections = 0 392 for table in tables: 393 sql = "SELECT MIN(diffObjID), MAX(diffObjID), COUNT(diffObjID) FROM " + table 394 rs = self.scratchDb.executeQuery(sql) 395 rs.first() 396 self.totalDetections = self.totalDetections + rs.getLong(3) 397 if first: 398 self.minObjID = rs.getLong(1) 399 self.maxObjID = rs.getLong(2) 400 else: 401 if rs.getLong(1) < self.minObjID: self.minObjID = rs.getLong(1) 402 if rs.getLong(2) > self.maxObjID: self.maxObjID = rs.getLong(2) 403 first = False 404 rs.close() 405 self.ippToPspsDb.updateDetectionStats(self.batchID, self.minObjID, self.maxObjID, self.totalDetections) 406 self.logger.infoPair("Total detections", "%ld" % self.totalDetections) 407 self.logger.infoPair("Min objID", "%ld" % self.minObjID) 408 self.logger.infoPair("Max objID", "%ld" % self.maxObjID) 409 410 411 364 412 ''' 365 413 Creates PSPS tables from VOTable descriptions … … 373 421 self.tablesToExport.append(table.name) 374 422 375 return self.alterPspsTables();423 return True 376 424 377 425 ''' … … 383 431 384 432 ''' 385 Alter PSPS tables 433 Alter PSPS tables (Subclass implements this) 386 434 ''' 387 435 def alterPspsTables(self): … … 417 465 418 466 # print "read smf table ", table 467 self.logger.infoPair("read smf table", table) 419 468 420 469 # drop any previous tables before import … … 440 489 441 490 return True 442 '''443 Imports IPP tables from FITS file444 445 Accepts a regular expression filter so not all tables need to be imported446 '''447 def importIppSTTables(self, columns="*", filter=""):448 449 self.logger.infoPair("Importing ST tables with filter", filter)450 451 # print "trying to read ", self.fits.getPath()452 453 # ST has 5 cmf files - all with the same table names. To counter this we do it like this.454 455 filters = ["g","r","i","z","y"]456 count = 0457 for f in filters:458 skip = 0459 if (f == "g"):460 if self.gstackID > 0:461 fileName = self.gfits.getPath()462 else:463 skip = 1464 if (f == "r"):465 if self.rstackID > 0:466 fileName = self.rfits.getPath()467 else:468 skip = 1469 if (f == "i"):470 if self.istackID > 0:471 fileName = self.ifits.getPath()472 else:473 skip = 1474 if (f == "z"):475 if self.zstackID > 0:476 fileName = self.zfits.getPath()477 else:478 skip = 1479 if (f == "y"):480 if self.ystackID > 0:481 fileName = self.yfits.getPath()482 else:483 skip = 1484 485 if skip == 1:486 self.logger.infoPair("no files for filter" , f)487 else:488 self.logger.infoPair("using filename:",fileName)489 490 try:491 tables = stilts.treads(fileName)492 except:493 self.logger.errorPair("STILTS could not import from", fileName)494 return False495 496 #count = 0497 for table in tables:498 499 # print "import smf table ", table500 match = re.match(filter, table.name)501 502 if not match: continue503 self.logger.infoPair("Reading IPP table", f + table.name)504 table = stilts.tpipe(table, cmd='addcol table_index $0')505 506 table = stilts.tpipe(table, cmd='explodeall')507 508 # print "read smf table ", table509 510 # drop any previous tables before import511 self.scratchDb.dropTable(f + table.name)512 513 # IPP FITS files are littered with infinities, so remove these514 self.logger.debug("Removing Infinity values from all columns")515 table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')516 table = stilts.tpipe(table, cmd='replaceval -Infinity null *')517 table = stilts.tpipe(table, cmd='replaceval Infinity null *')518 # print "cleaned up values ", table519 520 try:521 table.write(self.scratchDb.url + '#' + f + table.name)522 # XXX this one is not needed523 # self.scratchDb.killLastConnectionID()524 count = count + 1525 except:526 self.logger.exception("Problem writing table '" + f + table.name + "' to the database")527 528 self.logger.infoPair("Done. Imported", "%d tables" % count)529 self.indexIppTables()530 531 return True532 491 533 492 ''' … … 548 507 # get everything from table 549 508 try: 550 print "reading table from mysql ", table551 509 _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) 552 510 self.scratchDb.killLastConnectionID() … … 633 591 def run(self): 634 592 635 # this is badly named : it creates the tables and then fills in 636 # the ImageMeta tables for each ota (alterPspsTables) 593 # creates the empty PSPS tables 637 594 if not self.createEmptyPspsTables(): 638 595 self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables") … … 640 597 raise 641 598 599 # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch) 600 if not self.alterPspsTables(): 601 self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type") 602 self.ippToPspsDb.updateProcessed(self.batchID, -1) 603 raise 604 642 605 # for P2/ST, this reads the detection tables from the CMF/SMF file(s) 643 606 # for OB, this imports object data from DVO 644 if self.batchType == "ST": 645 self.logger.infoPair("did we get here","?") 646 if not self.importIppSTTables(): 647 self.logger.errorPair("skipping this batch", "could not import ST IPP tables") 648 self.ippToPspsDb.updateProcessed(self.batchID, -1) 649 raise 650 self.logger.infoPair("we did the ST","yes") 651 else: 652 # if not ST we do this 653 if not self.importIppTables(): 654 if self.batchType == "OB": 655 self.logger.errorPair("skipping this batch", "no dvo tables for this region") 656 self.ippToPspsDb.updateProcessed(self.batchID, 1) 657 return True 658 self.logger.errorPair("Aborting this batch", "could not import IPP tables") 659 self.ippToPspsDb.updateProcessed(self.batchID, -1) 660 raise 607 if not self.importIppTables(): 608 if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO"): 609 self.logger.errorPair("skipping this batch", "no dvo tables for this region") 610 self.ippToPspsDb.updateProcessed(self.batchID, 1) 611 return True 612 self.logger.errorPair("Aborting this batch", "could not import IPP tables") 613 self.ippToPspsDb.updateProcessed(self.batchID, -1) 614 raise 615 661 616 self.logger.infoPair("populatePspsTables","ok") 662 617 if not self.populatePspsTables(): … … 692 647 tarballFile) 693 648 649 if self.config.onebatch: 650 self.logger.infoPair("onebatch requested", "exiting") 651 sys.exit(0) 652 653 # XXX why is this down here?? 694 654 from datastore import Datastore 695
Note:
See TracChangeset
for help on using the changeset viewer.
