Changeset 37246 for trunk/ippToPsps/jython/batch.py
- Timestamp:
- Aug 12, 2014, 4:10:18 PM (12 years ago)
- File:
-
- 1 edited
-
trunk/ippToPsps/jython/batch.py (modified) (16 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps/jython/batch.py
r36726 r37246 37 37 batchID, 38 38 batchType, 39 fits, 40 useFullTables): 41 42 # print "starting the batch" 43 44 # self.printline = 0 45 # self.testprint() 39 fits): 46 40 47 41 self.readHeader = False … … 49 43 self.skychunk = skychunk 50 44 self.fits = fits 51 self.useFullTables = useFullTables52 45 53 46 # set up logging … … 55 48 self.logger.infoSeparator() 56 49 self.logger.debug("Batch class constructor") 57 58 # self.testprint()59 50 60 51 # set up class variables … … 68 59 self.pspsVoTableFilePath = self.config.configDir + "tables." + batchType + ".vot" 69 60 70 # self.testprint()71 72 61 # check FITS file is ok 73 62 if self.fits: … … 79 68 raise 80 69 81 self.scratchDb.setUseFullTables(self.useFullTables) 70 # define the dvo table names 71 self.scratchDb.setDvoTableNames() 82 72 83 73 # TODO … … 107 97 if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) 108 98 109 # self.testprint()110 111 99 # store today's date 112 100 now = datetime.datetime.now(); 113 101 self.dateStr = now.strftime("%Y-%m-%d") 114 102 115 # create DVO tables if accessing DVO directly116 if not self.useFullTables:self.scratchDb.createDvoTables()103 # create DVO tables (scratch database) 104 self.scratchDb.createDvoTables() 117 105 118 106 # dump stuff to the log … … 120 108 self.logger.infoPair("Batch name", self.batchName) 121 109 self.logger.infoPair("Survey ID", "%d" % self.surveyID) 122 self.logger.infoBool("Use full DVO tables?", self.useFullTables)123 110 124 111 if self.fits: 125 112 self.logger.infoPair("Input FITS file", self.fits.getOriginalPath()) 126 113 self.logger.infoPair("Input FITS primary header", "%s cards found" % self.fits.getPrimaryHeaderCardCount()) 127 128 # self.testprint()129 114 130 115 self.logger.infoPair("Output path", self.localOutPath) … … 237 222 self.logger.errorPair("Missing header field", key) 238 223 return "NULL" 224 225 ''' 226 Returns the string keyword value from this header or else "NULL" 227 ''' 228 def getKeyValue(self, header, key): 229 230 if key in header: return header[key] 231 else: 232 self.logger.errorPair("Missing header field", key) 233 return "NULL" 234 235 ''' 236 Returns the string keyword value from this header or else "NULL" 237 ''' 238 def getKeyFloat(self, header, format, key): 239 240 if key in header: 241 value = format % float(header[key]) 242 else: 243 self.logger.errorPair("Missing header field", key) 244 value = format % -999.9 245 246 return value 247 248 ''' 249 Returns the string keyword value from this header or else "NULL" 250 ''' 251 def getKeyInt(self, header, defValue, key): 252 253 if key in header: 254 value = str(int(header[key])) 255 else: 256 self.logger.errorPair("Missing header field", key) 257 value = str(defValue) 258 259 return value 239 260 240 261 ''' … … 373 394 self.tablesToExport.append(table.name) 374 395 375 return self.alterPspsTables();396 return True 376 397 377 398 ''' … … 383 404 384 405 ''' 385 Alter PSPS tables 406 Alter PSPS tables (Subclass implements this) 386 407 ''' 387 408 def alterPspsTables(self): … … 417 438 418 439 # print "read smf table ", table 440 self.logger.infoPair("read smf table", table) 419 441 420 442 # drop any previous tables before import … … 440 462 441 463 return True 442 '''443 Imports IPP tables from FITS file444 445 Accepts a regular expression filter so not all tables need to be imported446 '''447 def importIppSTTables(self, columns="*", filter=""):448 449 self.logger.infoPair("Importing ST tables with filter", filter)450 451 # print "trying to read ", self.fits.getPath()452 453 # ST has 5 cmf files - all with the same table names. To counter this we do it like this.454 455 filters = ["g","r","i","z","y"]456 count = 0457 for f in filters:458 skip = 0459 if (f == "g"):460 if self.gstackID > 0:461 fileName = self.gfits.getPath()462 else:463 skip = 1464 if (f == "r"):465 if self.rstackID > 0:466 fileName = self.rfits.getPath()467 else:468 skip = 1469 if (f == "i"):470 if self.istackID > 0:471 fileName = self.ifits.getPath()472 else:473 skip = 1474 if (f == "z"):475 if self.zstackID > 0:476 fileName = self.zfits.getPath()477 else:478 skip = 1479 if (f == "y"):480 if self.ystackID > 0:481 fileName = self.yfits.getPath()482 else:483 skip = 1484 485 if skip == 1:486 self.logger.infoPair("no files for filter" , f)487 else:488 self.logger.infoPair("using filename:",fileName)489 490 try:491 tables = stilts.treads(fileName)492 except:493 self.logger.errorPair("STILTS could not import from", fileName)494 return False495 496 #count = 0497 for table in tables:498 499 # print "import smf table ", table500 match = re.match(filter, table.name)501 502 if not match: continue503 self.logger.infoPair("Reading IPP table", f + table.name)504 table = stilts.tpipe(table, cmd='addcol table_index $0')505 506 table = stilts.tpipe(table, cmd='explodeall')507 508 # print "read smf table ", table509 510 # drop any previous tables before import511 self.scratchDb.dropTable(f + table.name)512 513 # IPP FITS files are littered with infinities, so remove these514 self.logger.debug("Removing Infinity values from all columns")515 table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')516 table = stilts.tpipe(table, cmd='replaceval -Infinity null *')517 table = stilts.tpipe(table, cmd='replaceval Infinity null *')518 # print "cleaned up values ", table519 520 try:521 table.write(self.scratchDb.url + '#' + f + table.name)522 # XXX this one is not needed523 # self.scratchDb.killLastConnectionID()524 count = count + 1525 except:526 self.logger.exception("Problem writing table '" + f + table.name + "' to the database")527 528 self.logger.infoPair("Done. Imported", "%d tables" % count)529 self.indexIppTables()530 531 return True532 464 533 465 ''' … … 548 480 # get everything from table 549 481 try: 550 print "reading table from mysql ", table551 482 _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) 552 483 self.scratchDb.killLastConnectionID() … … 633 564 def run(self): 634 565 635 # this is badly named : it creates the tables and then fills in 636 # the ImageMeta tables for each ota (alterPspsTables) 566 # creates the empty PSPS tables 637 567 if not self.createEmptyPspsTables(): 638 568 self.logger.errorPair("Aborting this batch", "could not create empty PSPS tables") … … 640 570 raise 641 571 572 # fill in basic, not cmf/smf data (eg, ImageMeta for detectionbatch) 573 if not self.alterPspsTables(): 574 self.logger.errorPair("Aborting this batch", "could not alter PSPS tables for batch type") 575 self.ippToPspsDb.updateProcessed(self.batchID, -1) 576 raise 577 642 578 # for P2/ST, this reads the detection tables from the CMF/SMF file(s) 643 579 # for OB, this imports object data from DVO 644 if self.batchType == "ST": 645 self.logger.infoPair("did we get here","?") 646 if not self.importIppSTTables(): 647 self.logger.errorPair("skipping this batch", "could not import ST IPP tables") 648 self.ippToPspsDb.updateProcessed(self.batchID, -1) 649 raise 650 self.logger.infoPair("we did the ST","yes") 651 else: 652 # if not ST we do this 653 if not self.importIppTables(): 654 if self.batchType == "OB": 655 self.logger.errorPair("skipping this batch", "no dvo tables for this region") 656 self.ippToPspsDb.updateProcessed(self.batchID, 1) 657 return True 658 self.logger.errorPair("Aborting this batch", "could not import IPP tables") 659 self.ippToPspsDb.updateProcessed(self.batchID, -1) 660 raise 580 if not self.importIppTables(): 581 if self.batchType == "OB": 582 self.logger.errorPair("skipping this batch", "no dvo tables for this region") 583 self.ippToPspsDb.updateProcessed(self.batchID, 1) 584 return True 585 self.logger.errorPair("Aborting this batch", "could not import IPP tables") 586 self.ippToPspsDb.updateProcessed(self.batchID, -1) 587 raise 588 661 589 self.logger.infoPair("populatePspsTables","ok") 662 590 if not self.populatePspsTables(): … … 692 620 tarballFile) 693 621 622 if self.config.onebatch: 623 self.logger.infoPair("onebatch requested", "exiting") 624 sys.exit(0) 625 626 # XXX why is this down here?? 694 627 from datastore import Datastore 695
Note:
See TracChangeset
for help on using the changeset viewer.
