Index: /branches/czw_branch/20160809/ippToPsps/config/Makefile.am =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/Makefile.am (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/config/Makefile.am (revision 39922) @@ -10,4 +10,5 @@ tables.FO.vot \ tables.FG.vot \ + tables.GO.vot \ settings.xml Index: /branches/czw_branch/20160809/ippToPsps/config/changes.txt =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/changes.txt (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/config/changes.txt (revision 39922) @@ -9,2 +9,11 @@ -- not yet alter table batch add column smf_version varchar(16) after comment; + +alter table skychunk add column queue_FG tinyint after queue_FO; +alter table skychunk add column queue_GO tinyint after queue_FG; + +alter table skychunk change column queue_FG queue_FG tinyint(1); +alter table skychunk change column queue_GO queue_GO tinyint(1); +alter table skychunk change column queue_GO queue_GO tinyint(1) default 0; +alter table skychunk change column queue_FO queue_FO tinyint(1) default 0; +alter table skychunk change column queue_FG queue_FG tinyint(1) default 0; Index: /branches/czw_branch/20160809/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/ippToPspsDbSchema.sql (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/config/ippToPspsDbSchema.sql (revision 39922) @@ -111,4 +111,6 @@ `queue_FW` tinyint(1) default '0', `queue_FO` tinyint(1) default '0', + `queue_FG` tinyint(1) default '0', + `queue_GO` tinyint(1) default '0', `active` tinyint(1) default '1', `parallel` tinyint(1) default '0', @@ -217,4 +219,34 @@ ) ENGINE=InnoDB DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; +-- +-- Table structure for table `forcedgalobject` +-- +DROP TABLE IF EXISTS `forcedgalobject`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `forcedgalobject` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `forcedgalobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + + + +-- +-- Table structure for table `gaiaobject` +-- +DROP TABLE IF EXISTS `gaiaobject`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `gaiaobject` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `gaiaobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + -- Index: /branches/czw_branch/20160809/ippToPsps/config/schema_browser.vot =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/schema_browser.vot (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/config/schema_browser.vot (revision 39922) @@ -171,4 +171,6 @@ These tables contain values that are derived from the fundamantal data products. +
+
Index: /branches/czw_branch/20160809/ippToPsps/config/settings.xml =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/settings.xml (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/config/settings.xml (revision 39922) @@ -819,5 +819,5 @@ - ipptopsps2 + ipptopsps3 ipp006 dvo Index: /branches/czw_branch/20160809/ippToPsps/config/tables.GO.vot =================================================================== --- /branches/czw_branch/20160809/ippToPsps/config/tables.GO.vot (revision 39922) +++ /branches/czw_branch/20160809/ippToPsps/config/tables.GO.vot (revision 39922) @@ -0,0 +1,43 @@ + + + + + + + PSPS objects calibrated against GAIA astrometry + + + + Unique object identifier. + + + Unique internal PSPS object identifier. + + + IPP internal object identifier. + + + Internal database batch identifier. + + + Information flag bitmask. + + + Right ascension from single epoch detections (weighted mean) in equinox J2000 at the mean epoch given by epochMean and calibrated against GAIA. + + + Declination from single epoch detections (weighted mean) in equinox J2000 at the mean epoch given by epochMean and calibrated against GAIA. + + + Right ascension standard deviation from single epoch detections. + + + Declination standard deviation from single epoch detections. + + + + +
+ + Index: /branches/czw_branch/20160809/ippToPsps/jython/Makefile.am =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/Makefile.am (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/Makefile.am (revision 39922) @@ -19,4 +19,5 @@ dvoforcedgalaxy.py \ dvoobjects.py \ + dvogaiaobjects.py \ dvo.py \ dxlayer.py \ @@ -25,4 +26,5 @@ forcedwarpbatch.py \ forcedgalaxybatch.py \ + gaiaobjectbatch.py \ gpc1db.py \ initbatch.py \ Index: /branches/czw_branch/20160809/ippToPsps/jython/batch.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/batch.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/batch.py (revision 39922) @@ -700,5 +700,5 @@ # for OB, this imports object data from DVO if not self.importIppTables(): - if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG"): + if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG" or self.batchType =="GO"): self.logger.errorPair("skipping this batch", "no dvo tables for this region") self.ippToPspsDb.updateProcessed(self.batchID, 1) Index: /branches/czw_branch/20160809/ippToPsps/jython/cleanup.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/cleanup.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/cleanup.py (revision 39922) @@ -51,4 +51,5 @@ self.clean("FO") self.clean("FG") + self.clean("GO") self.checkClientStatus() Index: /branches/czw_branch/20160809/ippToPsps/jython/dvo.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/dvo.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/dvo.py (revision 39922) @@ -152,4 +152,5 @@ self.scratchDb.execute("ALTER TABLE " + self.scratchDb.dvoImagesTable + " ADD PRIMARY KEY (IMAGE_ID, EXTERN_ID)") + self.scratchDb.execute("CREATE INDEX externID_index ON dvoImagesFull (EXTERN_ID)") self.scratchDb.setImportedThisDvoTable(path) return True Index: /branches/czw_branch/20160809/ippToPsps/jython/dvogaiaobjects.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/dvogaiaobjects.py (revision 39922) +++ /branches/czw_branch/20160809/ippToPsps/jython/dvogaiaobjects.py (revision 39922) @@ -0,0 +1,68 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoGaiaObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): +# def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoGaiaObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) +# super(DvoGaiaObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpt', 'cps'] + self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches + + ''' + ingest object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + # drop cpt/cps/object table? + # create detections table? + + # TODO path to DVO prog hardcoded temporarily + cmd = "dvopsps objects" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + # update lists after attempted sync + # self.printSummary() + + return True Index: /branches/czw_branch/20160809/ippToPsps/jython/fits.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/fits.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/fits.py (revision 39922) @@ -147,7 +147,8 @@ if False: + self.logger.infoPair("running findAndReadHeaderRaw",name) status = self.findAndReadHeaderRaw(name, VERBOSE) return status - + self.logger.infoPair("running findAndReadHeaderOhana",name) status = self.findAndReadHeaderOhana(name, VERBOSE) return status Index: /branches/czw_branch/20160809/ippToPsps/jython/gaiaobjectbatch.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/gaiaobjectbatch.py (revision 39922) +++ /branches/czw_branch/20160809/ippToPsps/jython/gaiaobjectbatch.py (revision 39922) @@ -0,0 +1,261 @@ +#!/usr/bin/env jython + +import os.path +import sys + +import stilts +from java.lang import * +from java.sql import * + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from dvogaiaobjects import DvoGaiaObjects +from sqlUtility import sqlUtility + +import logging.config + +''' +GaiaObjectBatch class + +This class, a sub-class of Batch, processes individual DVO region file pairs (cpt and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file. + +''' +class GaiaObjectBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID): + + super(GaiaObjectBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID, + "GO", + None) + + try: + self.dvoGaiaObjects = DvoGaiaObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb, self.gpc1Db) + except: + self.logger.errorPair("Unable to create instance of", "DvoObjects") + raise + + # create an output filename, which is {dvoINDEX}.FITS + self.outputFitsFile = "%08d.FITS" % self.id + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # dump stuff to log + self.logger.infoPair("DVO INDEX", "%d" % self.id) + + ''' + Overriden from batch base-class. We import from DVO directly for objects + ''' + def importIppTables(self, columns="*", filter=""): + + self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) + self.ippToPspsDb.insertGaiaObjectMeta(self.batchID, self.region) + self.dvoGaiaObjects.nativeIngestRegion(self.region) + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + if not self.scratchDb.tableExists(cptTableName): + return False + + return True + + ''' + Applies indexes to the PSPS tables + ''' + def alterPspsTables(self): + + self.scratchDb.execute("ALTER TABLE GaiaFrameCoordinate CHANGE dec_ `dec` double") + + return True + + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self): + # sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \ + # (select @r:=0) r, ObjectThin t) as foo using (objID) set \ + # uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)"; + + sql = "UPDATE GaiaFrameCoordinate set uniquePspsGOid = (("+str(self.batchID)+"*1000000000 ) + row)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + # since dvopsps is now used, no action is needed here + # self.logger.infoPair("Creating indexes on", "IPP tables") + + return True + + def populateGaiaObjectTable(self): + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + + #if False: + # self. + # XXX EAM 20140724 : this is probably wrong : flux measurements can be 0.0 or negative: please review + self.logger.info("Populating GaiaFrameCoordinate") + self.logger.info("Inserting objects from cpt file") + + # note "dec" is a reserved word in MySQL + # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems... + # INSERT IGNORE INTO ObjectThin + + sqlLine = sqlUtility("INSERT INTO GaiaFrameCoordinate (") + + sqlLine.group("objID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("ra", "RA_MEAN") + sqlLine.group("`dec`", "DEC_MEAN") + sqlLine.group("raErr", "RA_ERR") + sqlLine.group("decErr", "DEC_ERR") + sqlLine.group("gaiaFlag", "FLAGS") + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate Object table", sql) + return False + + # add row count columns so we can perform joins to get colors + self.logger.infoPair("Adding 'row' columns to", "GaiaFrameCoordinate and cps tables") + self.scratchDb.addRowCountColumn("GaiaFrameCoordinate", "row") + self.scratchDb.addRowCountColumn(cpsTableName, "row") + +## self.insertMeanMags(cpsTableName) +## this is the new version + if False: + # this chunk of code deletes objects which are out of ra,dec range for the table. + # this was a problem in an early version of DVO for cases where the astrometry went insane. + # this causes problems for the ra = 0,360 boundary (the test below does not handle that situation) + # and the restrictions below are poorly defined for the regions near the pole. + + # in any case, ObjectThin needs to maintain the same order + # as the cpt table until MeanObjects have been created or + # the join to the cps table will fail + + # XXX EAM 20140724 : is this necessary?? + # objects can have out of range ra dec in dvo - need to find and kill them at the end + + self.logger.infoPair("Determining", "ra/dec range") + + raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) + raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) + decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) + decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) + + self.logger.infoPair("R_MIN", raMin) + self.logger.infoPair("R_MAX", raMax) + self.logger.infoPair("D_MIN", decMin) + self.logger.infoPair("D_MAX", decMax) + #count out of range + + sql = "SELECT count(*) FROM GaiaFrameCoordinate where \ + GaiaFrameCoordinate.`dec` > " + str(decMax) + " \ + or GaiaFrameCoordinate.`dec` < " + str(decMin) + " \ + or GaiaFrameCoordinate.ra > " + str(raMax) + " \ + or GaiaFrameCoordinate.ra < " + str(raMin) + + rs = self.scratchDb.executeQuery(sql) + + rs.first() + nToDelete = rs.getInt(1) + + #delete out of range + + + sql = "DELETE FROM GaiaFrameCoordinate where \ + GaiaFrameCoordinate.`dec` > (" + str(decMax) + " + .0033) or \ + GaiaFrameCoordinate.`dec` < (" + str(decMin) + " - .0033) or \ + GaiaFrameCoordinate.ra > (" + str(raMax) + " + .0033) or \ + GaiaFrameCoordinate.ra < (" + str(raMin) + " - .0033)" + self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't cull outsiders from GaiaFrameCoordinate table", sql) + raise + + self.logger.infoPair("updatePspsUniqueIDs","start") + self.updatePspsUniqueIDs() + self.logger.infoPair("updatePspsUniqueIDs","end") + + self.logger.infoPair("Dropping row column from", "GaiaFrameCoordinate table") + self.scratchDb.dropColumn("GaiaFrameCoordinate", "row") + self.logger.infoPair("Dropped row column", "GaiaFrameCoordinate") + + self.setMinMaxObjID(["GaiaFrameCoordinate"]) + + return True + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateGaiaObjectTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + + ### XXX the code below first removes duplicate objID entries + ### from ObjectThin, then does the same for MeanObject. This + ### is a big problem: we have no guarantee that the surviving + ### rows are the correct matched rows. + + ### force objID uniqueness on *** ObjectThin *** + self.logger.infoPair("Forcing uniqueness on", "objID in GaiaFrameCoordinate table") + rowCountBefore = self.scratchDb.getRowCount("GaiaFrameCoordinate") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE GaiaFrameCoordinate ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("GaiaFrameCoordinate") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + # delete the cpt, cps tables from the scratch mysql (or we will run out of space) + self.dvoGaiaObjects.purgeRegion(self.region) + + return True Index: /branches/czw_branch/20160809/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/gpc1db.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/gpc1db.py (revision 39922) @@ -659,4 +659,26 @@ + ''' + Gets the exp_id from teh exp_name + ''' + def getExpIdFromExpName(self, expName): + sql = "SELECT exp_id from rawExp where exp_name = '"+expName+"'" + try: + rs = self.executeQuery(sql) + except: + self.logger.infoPair("failed sql:",sql) + self.logger.exception("Can't query for imageIDs") + + + expIDs = [] + while (rs.next()): + expIDs.append(rs.getString(1)) + rs.close() + expID = expIDs[0] + return expID + + + + ''' Index: /branches/czw_branch/20160809/ippToPsps/jython/insert.dvodb.info.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/insert.dvodb.info.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/insert.dvodb.info.py (revision 39922) @@ -83,5 +83,5 @@ # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo #for object-like batches, info comes from dvo - if (batchType == "OB" or batchType =="DO" or batchType =="FO"): + if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"): self.dvoObjects.setSkyArea( self.skychunk.minRa, Index: /branches/czw_branch/20160809/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/ipptopspsdb.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/ipptopspsdb.py (revision 39922) @@ -819,4 +819,19 @@ sql = "INSERT INTO diffobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + ''' + Inserts some gaiaobject metadata for this batch ID + ''' + def insertGaiaObjectMeta(self, batchID, region): + + sql = "INSERT INTO gaiaobject ( \ batch_id \ ,region \ @@ -1104,4 +1119,6 @@ ,queue_FW \ ,queue_FO \ + ,queue_FG \ + ,queue_GO \ ,parallel \ ,P2_smf_version \ @@ -1177,15 +1194,18 @@ if rs.getInt(23) == 1: self.skychunk.batchTypes.append("DO") if rs.getInt(24) == 1: self.skychunk.batchTypes.append("FW") - if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FG") +### if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FG") fix this when P2 is done if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO") + if rs.getInt(26) == 1: self.skychunk.batchTypes.append("FG") + if rs.getInt(27) == 1: self.skychunk.batchTypes.append("GO") + self.skychunk.force = True # TODO self.skychunk.parallel = False # TODO - if rs.getInt(26) == 1: self.skychunk.parallel = True - self.skychunk.P2_smf_version = rs.getString(27) + if rs.getInt(28) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(29) # options: "use_new", "not_reproc", "use_original" - self.skychunk.ST_cmf_version = rs.getString(28) + self.skychunk.ST_cmf_version = rs.getString(30) # options: not used? @@ -1212,8 +1232,8 @@ # print "typename: " + rsmd.getColumnTypeName(16) - self.skychunk.trange_start = rs.getString(29) + self.skychunk.trange_start = rs.getString(31) # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" - self.skychunk.trange_end = rs.getString(30) + self.skychunk.trange_end = rs.getString(32) # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" Index: /branches/czw_branch/20160809/ippToPsps/jython/loader.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/loader.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/loader.py (revision 39922) @@ -30,4 +30,6 @@ from forcedobjectbatch import ForcedObjectBatch from forcedgalaxybatch import ForcedGalaxyBatch +from gaiaobjectbatch import GaiaObjectBatch + # # @@ -302,4 +304,13 @@ id, batchID) + elif batchType == "GO": + batch = GaiaObjectBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID) batch.run() Index: /branches/czw_branch/20160809/ippToPsps/jython/metrics.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/metrics.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/metrics.py (revision 39922) @@ -53,4 +53,5 @@ rateFO = self.printStats("FO") rateFG = self.printStats("FG") + rateGO = self.printStats("GO") self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") @@ -76,4 +77,5 @@ pendFOProcessed = self.printTableRow(stages, "FO") pendFGProcessed = self.printTableRow(stages, "FG") + pendGOProcessed = self.printTableRow(stages, "GO") self.writeTableSeparator(colCount) @@ -93,5 +95,6 @@ try: self.logger.infoPair("Estimated time for FOs", "%.1f hours" % (pendFOProcessed / rateFO)) except: pass - try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG)) + try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG)) + try: self.logger.infoPair("Estimated time for GOs", "%.1f hours" % (pendGOProcessed / rateGO)) except: pass @@ -107,4 +110,6 @@ self.plot.createDensityPlot("FO", True) self.plot.createDensityPlot("FG", True) + self.plot.createDensityPlot("GO", True) + # some clean-up before going to sleep self.ippToPspsDb.purgeDeadClients() Index: /branches/czw_branch/20160809/ippToPsps/jython/plotter.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/plotter.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/plotter.py (revision 39922) @@ -32,4 +32,7 @@ self.plot.createDensityPlot("FW") self.plot.createDensityPlot("FO") + self.plot.createDensityPlot("FG") + self.plot.createDensityPlot("GO") + ''' Index: /branches/czw_branch/20160809/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/pollOdm.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/pollOdm.py (revision 39922) @@ -42,5 +42,5 @@ else: self.stages = [self.STAGE] - if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG'] + if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG', 'GO'] else: self.batchTypes = [self.BATCHTYPE] @@ -126,5 +126,5 @@ ''' def printUsage(self): - super(PollOdm, self).printUsage(" [hours]") + super(PollOdm, self).printUsage(" [hours]") Index: /branches/czw_branch/20160809/ippToPsps/jython/queue.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/queue.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/queue.py (revision 39922) @@ -73,5 +73,5 @@ # for object-like batches, info comes from dvo - if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG"): + if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"): self.dvoObjects.setSkyArea( self.skychunk.minRa, Index: /branches/czw_branch/20160809/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/czw_branch/20160809/ippToPsps/jython/stackbatch.py (revision 39921) +++ /branches/czw_branch/20160809/ippToPsps/jython/stackbatch.py (revision 39922) @@ -129,12 +129,15 @@ self.zpImage[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_OBS') self.zpError[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_ERR') - - deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test) - if not deteffHeader: - if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) - return False - - self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') - self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) + hasdeteff = 1 + try: deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test) + except: + hasdeteff = 0 + #if not deteffHeader: + # if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter) + # return False + + if hasdeteff == 1: + self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF') + self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM')) ## MEH -- md5sum test @@ -733,6 +736,16 @@ # sersic fit has an extra parameter if (ippModelType == "PS_MODEL_SERSIC"): - sqlLine.group("a." + filter + model + "Nu", "1 / (2.0 * b.EXT_PAR_07)") - sqlLine.group("a." + filter + model + "NuErr", "'0.1'") # this is not measured at the moment + testSQL = 'show columns FROM rSkyChip_xfit LIKE "EXT_PAR_07"' + have_ext_par_07 = 1 + try: + rs = self.scratchDb.execute(sql) + except: + have_ext_par_07 = 0 + if have_ext_par_07 == 1: + sqlLine.group("a." + filter + model + "Nu", "1 / (2.0 * b.EXT_PAR_07)") + sqlLine.group("a." + filter + model + "NuErr", "'0.1'") # this is not measured at the moment + + ### I think, if ippModelType = PS_MODEL_SERSIC, and EXT_PAR_07 does not exist, then Nu, Nerr = null + # missing these two fields? @@ -1170,7 +1183,18 @@ # search the words for the o....g....o part for j in range(len(words)): - match = re.search('o\d\d\d\dg\d\d\d\d\o', words[j]) - if (match != None): + match1 = re.search('^\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading o: then ones with borked no os and cs will work now + match2 = re.search('\o\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading o: then ones with borked no os and cs will work now + match3 = re.search('\c\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading o: then ones with borked no os and cs will work now + if (match1 != None): ogo=j + ogoname = 'o'+words[ogo] + break + if (match2 != None): + ogo=j + ogoname = words[ogo] + break + if (match3 != None): + ogo=j + ogoname = words[ogo] break # did we find it? if so, are there are least 2 more elements past the ogo? @@ -1178,11 +1202,19 @@ if ogo > -1: - if (ogo +1) < len(words): + if (ogo +2) < len(words): #this was the mistake before # there is a o....g....o element, and the next 2 exist frameID = words[ogo+1] - else: - self.logger.infoPair("unable to find exp_id from", inputName) - raise + else: + #ogoname = "o"+words[ogo] + try: frameID = self.gpc1Db.getExpIdFromExpName(ogoname) + except: + self.logger.error("cant get expid") + raise + +# frameID = str(rs.getInt(1)) + +# self.logger.infoPair("unable to find exp_id from", inputName) +# raise # and then raise an error and stop the program. I expect this to mostly work fine # for the extra long label (?) - can probably get it from just the o g o (if exp_id @@ -1216,4 +1248,5 @@ self.tablesToExport.append("StackToFrame") + return True ''' @@ -1496,5 +1529,5 @@ if self.stackIDs[filter] > 0: self.populateStackMeta(filter) - + self.populateStackObjectThin() @@ -1529,8 +1562,16 @@ self.logger.infoPair("populating","StackToImage") self.populateStackToImage() - + + + self.logger.infoPair("populating","StackToFrame") - self.populateStackToFrame() - + +# self.populateStackToFrame() + if not self.populateStackToFrame(): + self.logger.errorPair("can't populate"," stack to frame") + return False + self.logger.infoPair("removing frameID"," duplicates") + self.scratchDb.execute("ALTER IGNORE TABLE StackToFrame ADD UNIQUE INDEX(frameID)") + self.setMinMaxObjID(["StackObjectThin"])