Index: /trunk/ippToPsps/config/Makefile.am =================================================================== --- /trunk/ippToPsps/config/Makefile.am (revision 39784) +++ /trunk/ippToPsps/config/Makefile.am (revision 39785) @@ -10,4 +10,5 @@ tables.FO.vot \ tables.FG.vot \ + tables.GO.vot \ settings.xml Index: /trunk/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 39784) +++ /trunk/ippToPsps/config/ippToPspsDbSchema.sql (revision 39785) @@ -111,4 +111,6 @@ `queue_FW` tinyint(1) default '0', `queue_FO` tinyint(1) default '0', + `queue_FG` tinyint(1) default '0', + `queue_GO` tinyint(1) default '0', `active` tinyint(1) default '1', `parallel` tinyint(1) default '0', Index: /trunk/ippToPsps/config/tables.GO.vot =================================================================== --- /trunk/ippToPsps/config/tables.GO.vot (revision 39785) +++ /trunk/ippToPsps/config/tables.GO.vot (revision 39785) @@ -0,0 +1,47 @@ + + + + + + + PSPS objects calibrated against GAIA astrometry + + + + Unique object identifier. + + + Unique internal PSPS object identifier. + + + IPP internal object identifier. + + + Internal database batch identifier. + + + Information flag bitmask. + + + Right ascension from single epoch detections (weighted + mean) in equinox J2000 at the mean epoch given by epochMean and + calibrated against GAIA. + + + Declination from single epoch detections (weighted mean) + in equinox J2000 at the mean epoch given by epochMean and calibrated + against GAIA. + + + Right ascension standard deviation from single epoch detections. + + + Declination standard deviation from single epoch detections. + + + + +
+ + Index: /trunk/ippToPsps/jython/batch.py =================================================================== --- /trunk/ippToPsps/jython/batch.py (revision 39784) +++ /trunk/ippToPsps/jython/batch.py (revision 39785) @@ -700,5 +700,5 @@ # for OB, this imports object data from DVO if not self.importIppTables(): - if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG"): + if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG" or self.batchType =="GO"): self.logger.errorPair("skipping this batch", "no dvo tables for this region") self.ippToPspsDb.updateProcessed(self.batchID, 1) Index: /trunk/ippToPsps/jython/cleanup.py =================================================================== --- /trunk/ippToPsps/jython/cleanup.py (revision 39784) +++ /trunk/ippToPsps/jython/cleanup.py (revision 39785) @@ -51,4 +51,5 @@ self.clean("FO") self.clean("FG") + self.clean("GO") self.checkClientStatus() Index: /trunk/ippToPsps/jython/dvogaiaobjects.py =================================================================== --- /trunk/ippToPsps/jython/dvogaiaobjects.py (revision 39785) +++ /trunk/ippToPsps/jython/dvogaiaobjects.py (revision 39785) @@ -0,0 +1,68 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from dvo import Dvo + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoGaiaObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db): +# def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb): + + super(DvoGaiaObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb, gpc1Db) +# super(DvoGaiaObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb) + + # declare DVO file types of interest + self.ingestFileTypes = ['cpt', 'cps'] + self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches + + ''' + ingest object data into MySQL database using the native DVO program dvopsps + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def nativeIngestRegion(self, region): + + # drop cpt/cps/object table? + # create detections table? + + # TODO path to DVO prog hardcoded temporarily + cmd = "dvopsps objects" + cmd += " -dbhost " + self.scratchDb.dbHost + cmd += " -dbname " + self.scratchDb.dbName + cmd += " -dbuser " + self.scratchDb.dbUser + cmd += " -dbpass " + self.scratchDb.dbPass + cmd += " -D CATDIR " + self.skychunk.dvoLocation + cmd += " -cpt " + region + + if self.skychunk.parallel: + if self.scratchDb.dbHost == "localhost": + self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb") + raise + + cmd += " -parallel" + + self.logger.infoPair("Running dvopsps", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + if p.returncode != 0: + self.logger.errorPair("dvopsps failed on ", region) + raise + + # update lists after attempted sync + # self.printSummary() + + return True Index: /trunk/ippToPsps/jython/gaiaobjectbatch.py =================================================================== --- /trunk/ippToPsps/jython/gaiaobjectbatch.py (revision 39785) +++ /trunk/ippToPsps/jython/gaiaobjectbatch.py (revision 39785) @@ -0,0 +1,258 @@ +#!/usr/bin/env jython + +import os.path +import sys + +import stilts +from java.lang import * +from java.sql import * + +from xml.etree.ElementTree import ElementTree, Element, tostring + +from batch import Batch +from gpc1db import Gpc1Db +from ipptopspsdb import IppToPspsDb +from scratchdb import ScratchDb +from dvoobjects import DvoObjects +from sqlUtility import sqlUtility + +import logging.config + +''' +GaiaObjectBatch class + +This class, a sub-class of Batch, processes individual DVO region file pairs (cpt and cps files). The cpt file contains a list of all objects in that region, the cps file contains the magnitudes and so has a row per filter per object (so, generally, has rows = 5 x number of objects). The Photcode.dat table us used to figure out the number and order of filters as they appear in the cps file. + +''' +class GaiaObjectBatch(Batch): + + ''' + Constructor + ''' + def __init__(self, + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID): + + super(GaiaObjectBatch, self).__init__( + logger, + config, + skychunk, + gpc1Db, + ippToPspsDb, + scratchDb, + dvoID, + batchID, + "OB", + None) + + try: + self.dvoGaiaObjects = DvoGaiaObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb, self.gpc1Db) + except: + self.logger.errorPair("Unable to create instance of", "DvoObjects") + raise + + # create an output filename, which is {dvoINDEX}.FITS + self.outputFitsFile = "%08d.FITS" % self.id + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # dump stuff to log + self.logger.infoPair("DVO INDEX", "%d" % self.id) + + ''' + Overriden from batch base-class. We import from DVO directly for objects + ''' + def importIppTables(self, columns="*", filter=""): + + self.region = self.scratchDb.getRegionNameFromThisDvoIndex(self.id) + self.ippToPspsDb.insertGaiaObjectMeta(self.batchID, self.region) + self.dvoGaiaObjects.nativeIngestRegion(self.region) + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + if not self.scratchDb.tableExists(cptTableName): + return False + + return True + + ''' + Applies indexes to the PSPS tables + ''' + def alterPspsTables(self): + + return True + + ''' + Updates table and generates pspsuniqueids + ''' + + def updatePspsUniqueIDs(self): + # sql = "UPDATE ObjectThin join (select @r:=@r+1 rownum, objID from \ + # (select @r:=0) r, ObjectThin t) as foo using (objID) set \ + # uniquePspsOBid = ((" +str(self.batchID) +"*1000000000 ) + rownum)"; + + sql = "UPDATE GaiaFrameCoordinate set uniquePspsGOid = (("+str(self.batchID)+"*1000000000 ) + row)" + try: self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + raise + + + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + # since dvopsps is now used, no action is needed here + # self.logger.infoPair("Creating indexes on", "IPP tables") + + return True + + def populateGaiaObjectTable(self): + + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + + if False: + # XXX EAM 20140724 : this is probably wrong : flux measurements can be 0.0 or negative: please review + self.logger.info("Populating GaiaFrameCoordinate") + self.logger.info("Inserting objects from cpt file") + + # note "dec" is a reserved word in MySQL + # XXX EAM 20140724 : do not use INGORE unless we discover unavoidable problems... + # INSERT IGNORE INTO ObjectThin + + sqlLine = sqlUtility("INSERT INTO GaiaFrameCoordinate (") + + sqlLine.group("objID", "EXT_ID") + sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") # NOTE: shift by 32 bits exactly + sqlLine.group("batchID", "'" + str(self.batchID) + "'") + sqlLine.group("ra", "RA_MEAN") + sqlLine.group("dec", "DEC_MEAN") + sqlLine.group("raErr", "RA_ERR") + sqlLine.group("decErr", "DEC_ERR") + + sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate Object table", sql) + return False + + # add row count columns so we can perform joins to get colors + self.logger.infoPair("Adding 'row' columns to", "GaiaFrameCoordinate and cps tables") + self.scratchDb.addRowCountColumn("GaiaFrameCoordinate", "row") + self.scratchDb.addRowCountColumn(cpsTableName, "row") + +## self.insertMeanMags(cpsTableName) +## this is the new version + if False: + # this chunk of code deletes objects which are out of ra,dec range for the table. + # this was a problem in an early version of DVO for cases where the astrometry went insane. + # this causes problems for the ra = 0,360 boundary (the test below does not handle that situation) + # and the restrictions below are poorly defined for the regions near the pole. + + # in any case, ObjectThin needs to maintain the same order + # as the cpt table until MeanObjects have been created or + # the join to the cps table will fail + + # XXX EAM 20140724 : is this necessary?? + # objects can have out of range ra dec in dvo - need to find and kill them at the end + + self.logger.infoPair("Determining", "ra/dec range") + + raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) + raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) + decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) + decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) + + self.logger.infoPair("R_MIN", raMin) + self.logger.infoPair("R_MAX", raMax) + self.logger.infoPair("D_MIN", decMin) + self.logger.infoPair("D_MAX", decMax) + #count out of range + + sql = "SELECT count(*) FROM gaiaFrameCoordinate where \ + gaiaFrameCoordinate.dec > " + str(decMax) + " \ + or gaiaFrameCoordinate.dec < " + str(decMin) + " \ + or gaiaFrameCoordinate.ra > " + str(raMax) + " \ + or gaiaFrameCoordinate.ra < " + str(raMin) + + rs = self.scratchDb.executeQuery(sql) + + rs.first() + nToDelete = rs.getInt(1) + + #delete out of range + + + sql = "DELETE FROM gaiaFrameCoordinate where \ + gaiaFrameCoordinate.dec > (" + str(decMax) + " + .0033) or \ + gaiaFrameCoordinate.dec < (" + str(decMin) + " - .0033) or \ + gaiaFrameCoordinate.ra > (" + str(raMax) + " + .0033) or \ + gaiaFrameCoordinate.ra < (" + str(raMin) + " - .0033)" + self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't cull outsiders from gaiaFrameCoordinate table", sql) + raise + + self.logger.infoPair("updatePspsUniqueIDs","start") + self.updatePspsUniqueIDs() + self.logger.infoPair("updatePspsUniqueIDs","end") + + self.logger.infoPair("Dropping row column from", "gaiaFrameCoordinate table") + self.scratchDb.dropColumn("gaiaFrameCoordinate", "row") + self.logger.infoPair("Dropped row column", "gaiaFrameCoordinate") + + self.setMinMaxObjID(["gaiaFrameCoordinate"]) + + return True + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + if not self.populateGaiaObjectTable(): return False + + # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number + + ### XXX the code below first removes duplicate objID entries + ### from ObjectThin, then does the same for MeanObject. This + ### is a big problem: we have no guarantee that the surviving + ### rows are the correct matched rows. + + ### force objID uniqueness on *** ObjectThin *** + self.logger.infoPair("Forcing uniqueness on", "objID in gaiaFrameCoordinate table") + rowCountBefore = self.scratchDb.getRowCount("gaiaFrameCoordinate") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE gaiaFrameCoordinate ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("gaiaFrameCoordinate") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + # delete the cpt, cps tables from the scratch mysql (or we will run out of space) + self.dvoGaiaObjects.purgeRegion(self.region) + + return True Index: /trunk/ippToPsps/jython/insert.dvodb.info.py =================================================================== --- /trunk/ippToPsps/jython/insert.dvodb.info.py (revision 39784) +++ /trunk/ippToPsps/jython/insert.dvodb.info.py (revision 39785) @@ -83,5 +83,5 @@ # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo #for object-like batches, info comes from dvo - if (batchType == "OB" or batchType =="DO" or batchType =="FO"): + if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"): self.dvoObjects.setSkyArea( self.skychunk.minRa, Index: /trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /trunk/ippToPsps/jython/ipptopspsdb.py (revision 39784) +++ /trunk/ippToPsps/jython/ipptopspsdb.py (revision 39785) @@ -819,4 +819,19 @@ sql = "INSERT INTO diffobject ( \ + batch_id \ + ,region \ + ) VALUES ( \ + " + str(batchID) + " \ + ,'" + region + "' \ + )" + + self.execute(sql) + + ''' + Inserts some gaiaobject metadata for this batch ID + ''' + def insertGaiaObjectMeta(self, batchID, region): + + sql = "INSERT INTO gaiaobject ( \ batch_id \ ,region \ @@ -1104,4 +1119,6 @@ ,queue_FW \ ,queue_FO \ + ,queue_FG \ + ,queue_GO \ ,parallel \ ,P2_smf_version \ @@ -1179,13 +1196,16 @@ ### if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FG") fix this when P2 is done if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO") + if rs.getInt(26) == 1: self.skychunk.batchTypes.append("FG") + if rs.getInt(27) == 1: self.skychunk.batchTypes.append("GO") + self.skychunk.force = True # TODO self.skychunk.parallel = False # TODO - if rs.getInt(26) == 1: self.skychunk.parallel = True - self.skychunk.P2_smf_version = rs.getString(27) + if rs.getInt(28) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(29) # options: "use_new", "not_reproc", "use_original" - self.skychunk.ST_cmf_version = rs.getString(28) + self.skychunk.ST_cmf_version = rs.getString(30) # options: not used? @@ -1212,8 +1232,8 @@ # print "typename: " + rsmd.getColumnTypeName(16) - self.skychunk.trange_start = rs.getString(29) + self.skychunk.trange_start = rs.getString(31) # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" - self.skychunk.trange_end = rs.getString(30) + self.skychunk.trange_end = rs.getString(32) # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00" Index: /trunk/ippToPsps/jython/loader.py =================================================================== --- /trunk/ippToPsps/jython/loader.py (revision 39784) +++ /trunk/ippToPsps/jython/loader.py (revision 39785) @@ -302,4 +302,13 @@ id, batchID) + elif batchType == "GO": + batch = GaiaObjectBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID) batch.run() Index: /trunk/ippToPsps/jython/metrics.py =================================================================== --- /trunk/ippToPsps/jython/metrics.py (revision 39784) +++ /trunk/ippToPsps/jython/metrics.py (revision 39785) @@ -53,4 +53,5 @@ rateFO = self.printStats("FO") rateFG = self.printStats("FG") + rateGO = self.printStats("GO") self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") @@ -76,4 +77,5 @@ pendFOProcessed = self.printTableRow(stages, "FO") pendFGProcessed = self.printTableRow(stages, "FG") + pendGOProcessed = self.printTableRow(stages, "GO") self.writeTableSeparator(colCount) @@ -93,5 +95,6 @@ try: self.logger.infoPair("Estimated time for FOs", "%.1f hours" % (pendFOProcessed / rateFO)) except: pass - try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG)) + try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG)) + try: self.logger.infoPair("Estimated time for GOs", "%.1f hours" % (pendGOProcessed / rateGO)) except: pass @@ -107,4 +110,6 @@ self.plot.createDensityPlot("FO", True) self.plot.createDensityPlot("FG", True) + self.plot.createDensityPlot("GO", True) + # some clean-up before going to sleep self.ippToPspsDb.purgeDeadClients() Index: /trunk/ippToPsps/jython/plotter.py =================================================================== --- /trunk/ippToPsps/jython/plotter.py (revision 39784) +++ /trunk/ippToPsps/jython/plotter.py (revision 39785) @@ -32,4 +32,7 @@ self.plot.createDensityPlot("FW") self.plot.createDensityPlot("FO") + self.plot.createDensityPlot("FG") + self.plot.createDensityPlot("GO") + ''' Index: /trunk/ippToPsps/jython/pollOdm.py =================================================================== --- /trunk/ippToPsps/jython/pollOdm.py (revision 39784) +++ /trunk/ippToPsps/jython/pollOdm.py (revision 39785) @@ -42,5 +42,5 @@ else: self.stages = [self.STAGE] - if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG'] + if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG', 'GO'] else: self.batchTypes = [self.BATCHTYPE] @@ -126,5 +126,5 @@ ''' def printUsage(self): - super(PollOdm, self).printUsage(" [hours]") + super(PollOdm, self).printUsage(" [hours]") Index: /trunk/ippToPsps/jython/queue.py =================================================================== --- /trunk/ippToPsps/jython/queue.py (revision 39784) +++ /trunk/ippToPsps/jython/queue.py (revision 39785) @@ -73,5 +73,5 @@ # for object-like batches, info comes from dvo - if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG"): + if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"): self.dvoObjects.setSkyArea( self.skychunk.minRa,