Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/changes.txt =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/changes.txt (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/changes.txt (revision 36680) @@ -0,0 +1,10 @@ +--these are because we do the update process + +alter table skychunk add column P2_smf_version varchar(16) after parallel; +alter table skychunk add column ST_cmf_version varchar(16) after P2_smf_version; + + +alter table skychunk add trange_start timestamp after ST_cmf_version; +alter table skychunk add trange_end timestamp after trange_start; + +-- not yet alter table batch add column smf_version varchar(16) after comment; Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/ippToPspsDbSchema.sql (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/ippToPspsDbSchema.sql (revision 36680) @@ -127,4 +127,8 @@ `active` tinyint(1) default '1', `parallel` tinyint(1) default '0', + `P2_smf_version` varchar(16) default NULL, + `ST_cmf_version` varchar(16) default NULL, + `trange_start` timestamp NOT NULL default '0000-00-00 00:00:00', + `trange_end` timestamp NOT NULL default '0000-00-00 00:00:00', UNIQUE KEY `name` (`name`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/settings.xml =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/settings.xml (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/settings.xml (revision 36680) @@ -43,4 +43,154 @@ dvo 16 + + + ipptopsps_scratch + ipp011 + dvo + dvo + 16 + + + + ipptopsps_scratch + ipp012 + dvo + dvo + 16 + + + + + + ipptopsps_scratch + ipp013 + dvo + dvo + 32 + + + + ipptopsps_scratch + ipp014 + dvo + dvo + 16 + + + + ipptopsps_scratch + ipp015 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp016 + dvo + dvo + 16 + + + ipptopsps_scratch + ipp018 + dvo + dvo + 16 + + + + ipptopsps_scratch + ipp019 + dvo + dvo + 16 + + + + ipptopsps_scratch + ipp020 + dvo + dvo + 16 + + + ipptopsps_scratch + ipp021 + dvo + dvo + 16 + + + + ipptopsps_scratch + ipp025 + dvo + dvo + 16 + + + + + + + ipptopsps_scratch + ipp037 + dvo + dvo + 16 + + + + + ipptopsps_scratch + ipp038 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp039 + dvo + dvo + 20 + + + + + ipptopsps_scratch + ipp040 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp041 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp042 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp043 + dvo + dvo + 20 + + + ipptopsps_scratch + ipp044 + dvo + dvo + 20 Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/Makefile.am =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/Makefile.am (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/Makefile.am (revision 36680) @@ -30,4 +30,5 @@ pslogger.py \ queue.py \ + removepending.py \ scratchdb.py \ setupScratchDb.py \ Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py (revision 36680) @@ -412,4 +412,6 @@ if not match: continue self.logger.debugPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + table = stilts.tpipe(table, cmd='explodeall') Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/cleanup.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/cleanup.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/cleanup.py (revision 36680) @@ -87,5 +87,5 @@ self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs)) self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs)) - + # self.logger.infoPair("Did I get here", "yes?") # delete stuff from local disk if self.skychunk.deleteLocal: @@ -110,11 +110,13 @@ self.reportResults("datastore", deleteFromDatastoreIDs, count) - + # self.logger.infoPair("here","here") # remove stuff from DXLayer if self.skychunk.deleteDxLayer: +# self.logger.infoPair ("skip remove from dxlayer", "HAF") count = 0 for id in deleteFromDxLayerIDs: - + self.logger.infoPair("delete from dxlayer", id); if self.dxlayer.deleteBatch(id): + self.logger.infoPair("delete dxlayer",id); self.ippToPspsDb.updateDeletedDxlayer(id, 1) count = count + 1 Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/detectionbatch.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/detectionbatch.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/detectionbatch.py (revision 36680) @@ -54,5 +54,5 @@ batchID, "P2", - gpc1Db.getCameraStageSmf(camID), + gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version), useFullTables) Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dvo.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dvo.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dvo.py (revision 36680) @@ -65,5 +65,5 @@ if not self.correctDvo: print "*******************************************************************************" - self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation) + self.logger.warning("switching to a new dvo: '" + self.skychunk.dvoLocation + "'") # response = raw_input("(y/n) ") # if response == "y": @@ -518,5 +518,7 @@ self.logger.debugPair("Reading IPP table", table.name) table = stilts.tpipe(table, cmd='explodeall') - + #adds an index to all the tables + # table = stilts.tpipe(table, cmd='addcol table_index $0') + # IPP FITS files are littered with infinity values. Remove them self.logger.debugPair("Removing", "infinity values") @@ -628,4 +630,17 @@ maxDEC = boxDim['maxDEC'] + dD_border + #find out if we use a time_range or not + valid_trange_start = 0 + valid_trange_end = 0 + use_trange = 0 + if (self.skychunk.trange_start != '0000-00-00 00:00:00'): + valid_trange_start = 1 + if (self.skychunk.trange_end != '0000-00-00 00:00:00'): + valid_trange_end = 1 + + if (valid_trange_start and valid_trange_end): + use_trange = 1 + + # TODO path to DVO prog hardcoded temporarily cmd = "dvopsps detections" @@ -644,4 +659,34 @@ cmd += " -parallel" + if use_trange: + # make tranges valid for dvopsps + time_start1 = self.skychunk.trange_start + time_end1 = self.skychunk.trange_end + time_start=re.sub('\s+',',',time_start1) + time_end=re.sub('\s+',',',time_end1) + + # make command + cmd += " -time-start " + time_start + cmd += " -time-end " + time_end + + useP2 = 0 + useST = 0 + + for batchType in self.skychunk.batchTypes: + self.logger.infoPair("batchType", batchType) + if batchType == "ST": + useST = 1 + if batchType == "P2": + useP2 = 1 + + if (useP2 ==0 and useST ==1): + #grab only stacks + cmd += " -photcode-start 11000 -photcode-end 11500" + + if (useP2 ==1 and useST==0): + #grabd only P2s + cmd += " -photcode-start 10000 -photcode-end 10577" + + self.logger.infoPair("Running dvopsps", cmd) p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dxlayer.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dxlayer.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dxlayer.py (revision 36680) @@ -25,4 +25,5 @@ ''' def deleteBatch(self, batchID): + #HAF This is really lazy to do this way, but for the time, will check both dx layers and try the first, if no go try the second. batch = Batch.getNameFromID(batchID) @@ -36,5 +37,29 @@ if p.returncode != 0: - self.logger.debugPair("DXLayer webservice", "failed") + self.logger.infoPair("DXLayer webservice", "failed") + return 0 + else: + self.logger.infoPair("DXLayer webservice", "successful") + + ret = self.decodeDXLayerResponse(tempFile.name) + + tempFile.close() + p.stdout.close() + + if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) + else: self.logger.infoPair("Unable to delete batch from DXLayer 1" , batch) + + + + + tempFile = tempfile.NamedTemporaryFile(mode='w+b') + + p = Popen("deleteFromDXLayer2.pl -b " + batch + " -f " + tempFile.name, + shell=True, + stdout=PIPE) + p.wait() + + if p.returncode != 0: + self.logger.infoPair("DXLayer webservice", "failed") return 0 else: @@ -46,8 +71,12 @@ p.stdout.close() - if ret: self.logger.infoPair("Deleted batch from DXLayer", batch) - else: self.logger.errorPair("Unable to delete batch from DXLayer", batch) + if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) + else: self.logger.infoPair("Unable to delete batch from DXLayer 2" , batch) + + + return ret + ''' Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/gpc1db.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/gpc1db.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/gpc1db.py (revision 36680) @@ -49,9 +49,9 @@ NB this uses Bill's potentially temporary 'skycells' database when finding RA/Dec for stacks. ''' - def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999): + def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999, tstart="0000-00-00 00:00:00", tend = "0000-00-00 00:00:00"): self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA)) self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec)) - + rows = [] @@ -80,5 +80,7 @@ AND addRun.state = 'full' \ AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ - AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" + AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \ + AND dateobs >= '" + tstart + "'\ + AND dateobs <= '" + tend + "'" elif batchType == "ST": @@ -91,5 +93,7 @@ JOIN mergedvodbRun USING(minidvodb_id) \ JOIN mergedvodbProcessed USING (merge_id) \ - JOIN stackRun USING(stack_id) JOIN skycell USING(skycell_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile using (stack_id) \ + JOIN skycell USING(skycell_id) \ WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ @@ -100,6 +104,8 @@ AND addRun.state = 'full' \ AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) - + AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND mjd_obs >= (to_days('" + tstart + "')-678941) \ + AND mjd_obs <= (to_days('" + tend + "') - 678941) " + self.logger.infoPair("sql",sql) try: # XXX EAM : test output @@ -212,6 +218,6 @@ Gets a camera-stage smf for this cam_id ''' - def getCameraStageSmf(self, camID): - + def getCameraStageSmf(self, camID, smfversion): + self.logger.infoPair("using", smfversion) self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID)) @@ -237,14 +243,89 @@ # list all smf files if a neb path files = [] - if path.startswith("neb"): - - # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? - f=os.popen("neb-ls -p "+path+"%smf") - for i in f.readlines(): - files.append(i.rstrip()) - - # or not a neb path - else: - files = glob.glob(path + ".smf") + + + #there are a couple of states for smfversion + + # "use_new" => we have reprocessed the data, use the .smf file + + # "not_reproc" => this has not been reprocessed, use the .smf file + + # "use_original" => we *may* have reprocessed the data, + # use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig + # use (file).smf if camRun.state = 'full' and if there is only a .smf + # fault out if camRun.state != 'full' -- that is a race condition we don't want. + + + if (smfversion == "not_reproc") or (smfversion == "use_new"): + if path.startswith("neb"): + + # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? + f=os.popen("neb-ls -p "+path+"%smf") + for i in f.readlines(): + files.append(i.rstrip()) + + # or not a neb path + else: + files = glob.glob(path + ".smf") + + if (smfversion == "use_original"): + # get state of camRun + sql = "SELECT state \ + FROM camProcessedExp \ + JOIN camRun USING(cam_id) \ + WHERE camRun.cam_id = %d" % camID + try: + rs = self.executeQuery(sql) + rs.first() + except: + self.logger.errorPair("Can't query for camRun.state for cam_id", str(camID)) + return None + + try: + # get state of camRun.cam_id + camrunstate = rs.getString(1) + except: + self.logger.errorPair("No state found for cam_id", str(camID) ) + return None + + if (camrunstate == "full"): + self.logger.infoPair("camRun.state is ",camrunstate) + # if full find smf files + if path.startswith("neb"): + + # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? + fsmf=os.popen("neb-ls -p "+path+"%smf") + fsmfs = fsmf.readlines() + fsmforig=os.popen("neb-ls -p "+path+"%smf.original") + fsmforigs = fsmforig.readlines() + smfCount = len(fsmfs) + smfOrigCount = len (fsmforigs) + + if (smfCount > 0 and smfOrigCount > 0): + #smf and original so use original + self.logger.infoPair("using:", path+"%smf.original") + for i in fsmforigs: + files.append(i.rstrip()) + + if (smfCount > 0 and smfOrigCount == 0): + self.logger.infoPair("using:", path+"%smf") + #smf file and no smf.original use .smf + for i in fsmfs: + files.append(i.rstrip()) + + #if not smf dont' find files ??) + # or not a neb path + + else: + # this is broken for non neb - on the other hand we probably won't do .original if so + files = glob.glob(path + ".smf") + + + + # if .smf and .original use .original + else: + self.logger.infoPair("camRun.state not full, can't ingest", str(camID) ) + return None + # XXX EAM : test output Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/ipptopspsdb.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/ipptopspsdb.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/ipptopspsdb.py (revision 36680) @@ -39,4 +39,6 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + AND comment = '" + self.skychunk.name + "' \ + AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ AND merged = 1 \ AND deleted_local = 0 \ @@ -65,4 +67,6 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + AND comment = '" + self.skychunk.name + "' \ + AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ AND purged = 1 \ AND deleted_local = 0 \ @@ -92,4 +96,6 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + AND comment = '" + self.skychunk.name + "' \ + AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ AND purged = 1 \ AND deleted_datastore = 0 \ @@ -119,5 +125,7 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ - AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ + AND comment = '" + self.skychunk.name + "' \ + AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ + AND (merge_worthy = 1) \ AND deleted_datastore = 0 AND block_delete_datastore = 0" @@ -144,5 +152,5 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ - AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ + AND (merge_worthy = 1) \ AND deleted_dxlayer = 0 and block_delete_dxlayer = 0" @@ -1109,4 +1117,8 @@ ,queue_OB \ ,parallel \ + ,P2_smf_version \ + ,ST_cmf_version \ + ,trange_start \ + ,trange_end \ FROM skychunk \ WHERE name = '" + self.skychunk.name + "'" @@ -1155,4 +1167,8 @@ if rs.getInt(22) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(23) + self.skychunk.ST_cmf_version = rs.getString(24) + self.skychunk.trange_start = rs.getString(25) + self.skychunk.trange_end = rs.getString(26) if self.skychunk.parallel: print "USING parallel" Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/odm.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/odm.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/odm.py (revision 36680) @@ -78,5 +78,5 @@ DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text results['DETAILS'] = DETAILS - elif re.search("processing", STATE): + elif re.search("processing", STATE): loadedToODM = 1 if re.search("MergeWorthy", MESSAGE): @@ -87,4 +87,12 @@ mergeWorthy = 1 merged = 1 +# if re.search("released", STATE): +# loadedToODM = 1 +# mergeWorthy = 1 +# merged = 1 +# if re.search("released", MESSAGE): +# loadedToODM = 1 +# mergeWorthy = 1 +# merged = 1 except: pass Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/queue.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/queue.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/queue.py (revision 36680) @@ -45,4 +45,5 @@ try: + print "why doesn't this work" self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) except: @@ -71,6 +72,6 @@ for batchType in self.skychunk.batchTypes: self.logger.infoPair("BatchType",batchType) - # self.logger.infoTitle("Previous failed datastore loads") - # self.publishAnyUnpublishedBatches(batchType) + #self.logger.infoTitle("Previous failed datastore loads") + #self.publishAnyUnpublishedBatches(batchType) # get totals for whole area to check if there is anything to do @@ -98,5 +99,7 @@ self.skychunk.maxRa, self.skychunk.minDec, - self.skychunk.maxDec) + self.skychunk.maxDec, + self.skychunk.trange_start, + self.skychunk.trange_end) # EAM TEST I/O Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/removepending.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/removepending.py (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/removepending.py (revision 36680) @@ -0,0 +1,86 @@ +#!/usr/bin/env jython + +# +# This is a way to easily delete pending skychunks +# + +import time +import sys +import os +import math +import logging.config + +from ipptopsps import IppToPsps +from gpc1db import Gpc1Db +from datastore import Datastore +from dvoobjects import DvoObjects +from batch import Batch +from scratchdb import ScratchDb +# basically, "ippjython removepending.py skychunk_name " will remove pending batches + + +''' +Queue class +''' +class Queue(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Queue, self).__init__(argv) + + # print "inited ipptopsps" + + # create various objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + # print "connect to gpc1" + + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) + + print "made datastore" + + # connect to scratch database + self.scratchDb = ScratchDb(self.logger, self.config) + + print "connect to scratchdb" + + try: + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.exitProgram("Unable to create instance of DvoObject") + raise + + print "got objects" + + self.skychunk.printAll() + + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + ''' + Main processing loop. + Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + by one or more loading clients that may be running on any host + ''' + def run(self): + + self.ippToPspsDb.removeAllBoxes() + print "remove boxes" + + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Queue, self).printUsage("[]") + + +''' +Start of program. +''' +try: + queue = Queue(sys.argv) + queue.run() + queue.exitProgram("completed") +except: pass + Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/stackbatch.py =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/stackbatch.py (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/stackbatch.py (revision 36680) @@ -243,5 +243,5 @@ cnt=cnt+1 - sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition + sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND "+psfCondition self.scratchDb.execute(sql) @@ -303,4 +303,50 @@ modelLong = model; + #does it have the EXT_COVAR in there? + + sql = "DESCRIBE SkyChip_xfit 'EXT_COVAR_%'" + rs = self.scratchDb.executeQuery(sql) + rs.first() + hasResult = rs.next() + if hasResult: + print "EXT_COVARs exist" + else: + print "No EXT_COVARS here" + + ext_covar_sql = "" + + if (hasResult): + ext_covar_sql = ", "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ + "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ + "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ + "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ + "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ + "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ + "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ + "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ + "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ + "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ + "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ + "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ + "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ + "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ + "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ + "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ + "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ + "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ + "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ + "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ + "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ + "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ + "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ + "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ + "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ + "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ + "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ + "+modelLong+"Covar77=b.EXT_COVAR_06_06" + + + + sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ "+model+"Radius=b.EXT_WIDTH_MAJ, \ @@ -308,39 +354,13 @@ "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \ "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ - "+model+"Phi=b.EXT_THETA, \ - "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ - "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ - "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ - "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ - "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ - "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ - "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ - "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ - "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ - "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ - "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ - "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ - "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ - "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ - "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ - "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ - "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ - "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ - "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ - "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ - "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ - "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ - "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ - "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ - "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ - "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ - "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ - "+modelLong+"Covar77=b.EXT_COVAR_06_06 \ + "+model+"Phi=b.EXT_THETA \ + "+ext_covar_sql+" \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" + print sql self.scratchDb.execute(sql) # sersic fit has an extra parameter - if ippModelType == "PS_MODEL_SERSIC": + if (ippModelType == "PS_MODEL_SERSIC" and hasResult > 0): sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ "+modelLong+"Covar18=b.EXT_COVAR_00_07, \ @@ -633,10 +653,14 @@ # TODO temporarily loading 1st convolved fluxes into unconvolved fields + + #This assumes that there are 3 different fluxs, the first is unconv, and the 2nd and 3rd are convolved. If there are more than 3 fluxes it will break. + # If there are more than 3 types of fluxes, we should do a count to find out how many there are and adjust this accordingly (make it more robust) + self.logger.infoPair("Adding fluxes", "un-convolved") - self.updateApFlxs("", "IS NULL") + self.updateApFlxs("", "((b.table_index -1) % 3 = 0)") self.logger.infoPair("Adding fluxes", "1st convolved") - self.updateApFlxs("c1", "< 7.0") + self.updateApFlxs("c1", "((b.table_index -1) % 3 = 1)") self.logger.infoPair("Adding fluxes", "2nd convolved") - self.updateApFlxs("c2", "> 7.0") + self.updateApFlxs("c2", "((b.table_index -1) % 3 = 2)") self.logger.infoPair("Adding", "petrosians for extended sources") @@ -645,5 +669,5 @@ ,petRadiusErr=b.PETRO_RADIUS_ERR \ ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \ - ,petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736 \ + ,petFluxErr=-999 \ ,petR50=b.PETRO_RADIUS_50 \ ,petR50Err=b.PETRO_RADIUS_50_ERR \ @@ -653,4 +677,15 @@ WHERE a.ippDetectID=b.IPP_IDET" self.scratchDb.execute(sql) + + self.logger.infoPair("Adding", "petFluxErr for non nulls") + sql = "UPDATE StackApFlx as a, SkyChip_xsrc AS b SET \ + petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736 \ + WHERE a.ippDetectID=b.IPP_IDET \ + AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)" + # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc) + self.logger.infoPair("sql", sql) + self.scratchDb.execute(sql) + + self.logger.infoPair("adding"," PrimaryF to stackApFlx") Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/Makefile.am =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/Makefile.am (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/Makefile.am (revision 36680) @@ -1,3 +1,3 @@ -bin_SCRIPTS = checkOdmStatus.pl convertPhotCodesToXml.pl deleteFromDXLayer.pl pspsSchema2xml.pl +bin_SCRIPTS = checkOdmStatus.pl convertPhotCodesToXml.pl deleteFromDXLayer.pl pspsSchema2xml.pl deleteFromDXLayer2.pl INSTALL = @INSTALL@ -C Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer.pl (revision 35859) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer.pl (revision 36680) @@ -18,5 +18,12 @@ ); -my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + +#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxwebservice.cgi"; +my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxlayer/dxwebservice.cgi"; + + +#my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + #my $dxWebServiceUrl = "http://166.122.172.121/cgi-bin/dxlayer/dxwebservice.cgi"; my $ua = LWP::UserAgent->new; Index: branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer2.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer2.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer2.pl (revision 36680) @@ -0,0 +1,66 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use LWP::UserAgent; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use XML::LibXML; +use File::Temp qw(tempfile); + + +my $batchName = undef; +my $file = undef; + +GetOptions( + 'batch|b=s' => \$batchName, + 'file|f=s' => \$file, + ); + +#my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + +#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxwebservice.cgi"; +my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + +#my $dxWebServiceUrl = "http://166.122.172.121/cgi-bin/dxlayer/dxwebservice.cgi"; +my $ua = LWP::UserAgent->new; +$ua->timeout(15); +$ua->env_proxy; + + my $doc = XML::LibXML::Document->new(); # Global document variable. + my $rootNode = $doc->createElement('AdminQueryParams'); +$doc->setDocumentElement($rootNode); + +my $paramNode = $doc->createElement('Param'); +$paramNode->setAttribute('Name', "BatchID"); # default FirstRow is 1 +my $text=XML::LibXML::Text->new($batchName); +$paramNode->appendChild($text); #append +$rootNode->appendChild($paramNode); # append a node to the xml tree + +my $xmlParams = $doc->toString; + +print $xmlParams; + +my $response = $ua->post($dxWebServiceUrl, + [op => "delete_dxbatch_files", + xmlParams => $xmlParams ] + ); + +# '200' is the 'OK' response +if ($response->code != 200) { + + print "Problem connecting to DXLayerweb service, HTTP response status: ".$response->status_line."\n"; + exit(1); +} + +open (DAT, ">".$file); +print DAT $response->content; + + +print "$file\n"; + +exit(0); + + +