Changeset 36680 for branches/eam_branches/ps2-tc3-20130727/ippToPsps
- Timestamp:
- Apr 21, 2014, 5:42:34 AM (12 years ago)
- Location:
- branches/eam_branches/ps2-tc3-20130727
- Files:
-
- 17 edited
- 3 copied
-
. (modified) (1 prop)
-
ippToPsps (modified) (1 prop)
-
ippToPsps/config/changes.txt (copied) (copied from trunk/ippToPsps/config/changes.txt )
-
ippToPsps/config/ippToPspsDbSchema.sql (modified) (1 diff)
-
ippToPsps/config/settings.xml (modified) (1 diff)
-
ippToPsps/jython/Makefile.am (modified) (1 diff)
-
ippToPsps/jython/batch.py (modified) (1 diff)
-
ippToPsps/jython/cleanup.py (modified) (2 diffs)
-
ippToPsps/jython/detectionbatch.py (modified) (1 diff)
-
ippToPsps/jython/dvo.py (modified) (4 diffs)
-
ippToPsps/jython/dxlayer.py (modified) (3 diffs)
-
ippToPsps/jython/gpc1db.py (modified) (6 diffs)
-
ippToPsps/jython/ipptopspsdb.py (modified) (7 diffs)
-
ippToPsps/jython/odm.py (modified) (2 diffs)
-
ippToPsps/jython/queue.py (modified) (3 diffs)
-
ippToPsps/jython/removepending.py (copied) (copied from trunk/ippToPsps/jython/removepending.py )
-
ippToPsps/jython/stackbatch.py (modified) (6 diffs)
-
ippToPsps/perl/Makefile.am (modified) (1 diff)
-
ippToPsps/perl/deleteFromDXLayer.pl (modified) (1 diff)
-
ippToPsps/perl/deleteFromDXLayer2.pl (copied) (copied from trunk/ippToPsps/perl/deleteFromDXLayer2.pl )
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ps2-tc3-20130727
- Property svn:mergeinfo changed
-
branches/eam_branches/ps2-tc3-20130727/ippToPsps
- Property svn:mergeinfo deleted
-
branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/ippToPspsDbSchema.sql
r35502 r36680 127 127 `active` tinyint(1) default '1', 128 128 `parallel` tinyint(1) default '0', 129 `P2_smf_version` varchar(16) default NULL, 130 `ST_cmf_version` varchar(16) default NULL, 131 `trange_start` timestamp NOT NULL default '0000-00-00 00:00:00', 132 `trange_end` timestamp NOT NULL default '0000-00-00 00:00:00', 129 133 UNIQUE KEY `name` (`name`) 130 134 ) ENGINE=InnoDB DEFAULT CHARSET=latin1; -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/config/settings.xml
r35656 r36680 43 43 <password>dvo</password> 44 44 <memory>16</memory> 45 </loader> 46 <loader mach="ipp011"> 47 <name>ipptopsps_scratch</name> 48 <host>ipp011</host> 49 <user>dvo</user> 50 <password>dvo</password> 51 <memory>16</memory> 52 </loader> 53 54 <loader mach="ipp012"> 55 <name>ipptopsps_scratch</name> 56 <host>ipp012</host> 57 <user>dvo</user> 58 <password>dvo</password> 59 <memory>16</memory> 60 </loader> 61 62 63 64 <loader mach="ipp013"> 65 <name>ipptopsps_scratch</name> 66 <host>ipp013</host> 67 <user>dvo</user> 68 <password>dvo</password> 69 <memory>32</memory> 70 </loader> 71 72 <loader mach="ipp014"> 73 <name>ipptopsps_scratch</name> 74 <host>ipp014</host> 75 <user>dvo</user> 76 <password>dvo</password> 77 <memory>16</memory> 78 </loader> 79 80 <loader mach="ipp015"> 81 <name>ipptopsps_scratch</name> 82 <host>ipp015</host> 83 <user>dvo</user> 84 <password>dvo</password> 85 <memory>20</memory> 86 </loader> 87 <loader mach="ipp016"> 88 <name>ipptopsps_scratch</name> 89 <host>ipp016</host> 90 <user>dvo</user> 91 <password>dvo</password> 92 <memory>16</memory> 93 </loader> 94 <loader mach="ipp018"> 95 <name>ipptopsps_scratch</name> 96 <host>ipp018</host> 97 <user>dvo</user> 98 <password>dvo</password> 99 <memory>16</memory> 100 </loader> 101 102 <loader mach="ipp019"> 103 <name>ipptopsps_scratch</name> 104 <host>ipp019</host> 105 <user>dvo</user> 106 <password>dvo</password> 107 <memory>16</memory> 108 </loader> 109 110 <loader mach="ipp020"> 111 <name>ipptopsps_scratch</name> 112 <host>ipp020</host> 113 <user>dvo</user> 114 <password>dvo</password> 115 <memory>16</memory> 116 </loader> 117 <loader mach="ipp021"> 118 <name>ipptopsps_scratch</name> 119 <host>ipp021</host> 120 <user>dvo</user> 121 <password>dvo</password> 122 <memory>16</memory> 123 </loader> 124 125 <loader mach="ipp025"> 126 <name>ipptopsps_scratch</name> 127 <host>ipp025</host> 128 <user>dvo</user> 129 <password>dvo</password> 130 <memory>16</memory> 131 </loader> 132 133 134 135 136 <loader mach="ipp037"> 137 <name>ipptopsps_scratch</name> 138 <host>ipp037</host> 139 <user>dvo</user> 140 <password>dvo</password> 141 <memory>16</memory> 142 </loader> 143 144 145 <loader mach="ipp038"> 146 <name>ipptopsps_scratch</name> 147 <host>ipp038</host> 148 <user>dvo</user> 149 <password>dvo</password> 150 <memory>20</memory> 151 </loader> 152 <loader mach="ipp039"> 153 <name>ipptopsps_scratch</name> 154 <host>ipp039</host> 155 <user>dvo</user> 156 <password>dvo</password> 157 <memory>20</memory> 158 </loader> 159 160 161 <loader mach="ipp040"> 162 <name>ipptopsps_scratch</name> 163 <host>ipp040</host> 164 <user>dvo</user> 165 <password>dvo</password> 166 <memory>20</memory> 167 </loader> 168 <loader mach="ipp041"> 169 <name>ipptopsps_scratch</name> 170 <host>ipp041</host> 171 <user>dvo</user> 172 <password>dvo</password> 173 <memory>20</memory> 174 </loader> 175 <loader mach="ipp042"> 176 <name>ipptopsps_scratch</name> 177 <host>ipp042</host> 178 <user>dvo</user> 179 <password>dvo</password> 180 <memory>20</memory> 181 </loader> 182 <loader mach="ipp043"> 183 <name>ipptopsps_scratch</name> 184 <host>ipp043</host> 185 <user>dvo</user> 186 <password>dvo</password> 187 <memory>20</memory> 188 </loader> 189 <loader mach="ipp044"> 190 <name>ipptopsps_scratch</name> 191 <host>ipp044</host> 192 <user>dvo</user> 193 <password>dvo</password> 194 <memory>20</memory> 45 195 </loader> 46 196 <loader mach="ipp045"> -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/Makefile.am
r35097 r36680 30 30 pslogger.py \ 31 31 queue.py \ 32 removepending.py \ 32 33 scratchdb.py \ 33 34 setupScratchDb.py \ -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/batch.py
r35496 r36680 412 412 if not match: continue 413 413 self.logger.debugPair("Reading IPP table", table.name) 414 table = stilts.tpipe(table, cmd='addcol table_index $0') 415 414 416 table = stilts.tpipe(table, cmd='explodeall') 415 417 -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/cleanup.py
r35311 r36680 87 87 self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs)) 88 88 self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs)) 89 89 # self.logger.infoPair("Did I get here", "yes?") 90 90 # delete stuff from local disk 91 91 if self.skychunk.deleteLocal: … … 110 110 111 111 self.reportResults("datastore", deleteFromDatastoreIDs, count) 112 112 # self.logger.infoPair("here","here") 113 113 # remove stuff from DXLayer 114 114 if self.skychunk.deleteDxLayer: 115 # self.logger.infoPair ("skip remove from dxlayer", "HAF") 115 116 count = 0 116 117 for id in deleteFromDxLayerIDs: 117 118 self.logger.infoPair("delete from dxlayer", id); 118 119 if self.dxlayer.deleteBatch(id): 120 self.logger.infoPair("delete dxlayer",id); 119 121 self.ippToPspsDb.updateDeletedDxlayer(id, 1) 120 122 count = count + 1 -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/detectionbatch.py
r35605 r36680 54 54 batchID, 55 55 "P2", 56 gpc1Db.getCameraStageSmf(camID ),56 gpc1Db.getCameraStageSmf(camID,skychunk.P2_smf_version), 57 57 useFullTables) 58 58 -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dvo.py
r35780 r36680 65 65 if not self.correctDvo: 66 66 print "*******************************************************************************" 67 self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation)67 self.logger.warning("switching to a new dvo: '" + self.skychunk.dvoLocation + "'") 68 68 # response = raw_input("(y/n) ") 69 69 # if response == "y": … … 518 518 self.logger.debugPair("Reading IPP table", table.name) 519 519 table = stilts.tpipe(table, cmd='explodeall') 520 520 #adds an index to all the tables 521 # table = stilts.tpipe(table, cmd='addcol table_index $0') 522 521 523 # IPP FITS files are littered with infinity values. Remove them 522 524 self.logger.debugPair("Removing", "infinity values") … … 628 630 maxDEC = boxDim['maxDEC'] + dD_border 629 631 632 #find out if we use a time_range or not 633 valid_trange_start = 0 634 valid_trange_end = 0 635 use_trange = 0 636 if (self.skychunk.trange_start != '0000-00-00 00:00:00'): 637 valid_trange_start = 1 638 if (self.skychunk.trange_end != '0000-00-00 00:00:00'): 639 valid_trange_end = 1 640 641 if (valid_trange_start and valid_trange_end): 642 use_trange = 1 643 644 630 645 # TODO path to DVO prog hardcoded temporarily 631 646 cmd = "dvopsps detections" … … 644 659 cmd += " -parallel" 645 660 661 if use_trange: 662 # make tranges valid for dvopsps 663 time_start1 = self.skychunk.trange_start 664 time_end1 = self.skychunk.trange_end 665 time_start=re.sub('\s+',',',time_start1) 666 time_end=re.sub('\s+',',',time_end1) 667 668 # make command 669 cmd += " -time-start " + time_start 670 cmd += " -time-end " + time_end 671 672 useP2 = 0 673 useST = 0 674 675 for batchType in self.skychunk.batchTypes: 676 self.logger.infoPair("batchType", batchType) 677 if batchType == "ST": 678 useST = 1 679 if batchType == "P2": 680 useP2 = 1 681 682 if (useP2 ==0 and useST ==1): 683 #grab only stacks 684 cmd += " -photcode-start 11000 -photcode-end 11500" 685 686 if (useP2 ==1 and useST==0): 687 #grabd only P2s 688 cmd += " -photcode-start 10000 -photcode-end 10577" 689 690 646 691 self.logger.infoPair("Running dvopsps", cmd) 647 692 p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/dxlayer.py
r35217 r36680 25 25 ''' 26 26 def deleteBatch(self, batchID): 27 #HAF This is really lazy to do this way, but for the time, will check both dx layers and try the first, if no go try the second. 27 28 28 29 batch = Batch.getNameFromID(batchID) … … 36 37 37 38 if p.returncode != 0: 38 self.logger.debugPair("DXLayer webservice", "failed") 39 self.logger.infoPair("DXLayer webservice", "failed") 40 return 0 41 else: 42 self.logger.infoPair("DXLayer webservice", "successful") 43 44 ret = self.decodeDXLayerResponse(tempFile.name) 45 46 tempFile.close() 47 p.stdout.close() 48 49 if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) 50 else: self.logger.infoPair("Unable to delete batch from DXLayer 1" , batch) 51 52 53 54 55 tempFile = tempfile.NamedTemporaryFile(mode='w+b') 56 57 p = Popen("deleteFromDXLayer2.pl -b " + batch + " -f " + tempFile.name, 58 shell=True, 59 stdout=PIPE) 60 p.wait() 61 62 if p.returncode != 0: 63 self.logger.infoPair("DXLayer webservice", "failed") 39 64 return 0 40 65 else: … … 46 71 p.stdout.close() 47 72 48 if ret: self.logger.infoPair("Deleted batch from DXLayer", batch) 49 else: self.logger.errorPair("Unable to delete batch from DXLayer", batch) 73 if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) 74 else: self.logger.infoPair("Unable to delete batch from DXLayer 2" , batch) 75 76 77 50 78 51 79 return ret 80 52 81 53 82 ''' -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/gpc1db.py
r35417 r36680 49 49 NB this uses Bill's potentially temporary 'skycells' database when finding RA/Dec for stacks. 50 50 ''' 51 def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999 ):51 def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999, tstart="0000-00-00 00:00:00", tend = "0000-00-00 00:00:00"): 52 52 53 53 self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA)) 54 54 self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec)) 55 55 56 56 rows = [] 57 57 … … 80 80 AND addRun.state = 'full' \ 81 81 AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ 82 AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" 82 AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \ 83 AND dateobs >= '" + tstart + "'\ 84 AND dateobs <= '" + tend + "'" 83 85 84 86 elif batchType == "ST": … … 91 93 JOIN mergedvodbRun USING(minidvodb_id) \ 92 94 JOIN mergedvodbProcessed USING (merge_id) \ 93 JOIN stackRun USING(stack_id) JOIN skycell USING(skycell_id) \ 95 JOIN stackRun USING(stack_id) \ 96 JOIN stackSumSkyfile using (stack_id) \ 97 JOIN skycell USING(skycell_id) \ 94 98 WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ 95 99 AND minidvodbRun.state = 'merged' \ … … 100 104 AND addRun.state = 'full' \ 101 105 AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ 102 AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) 103 106 AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ 107 AND mjd_obs >= (to_days('" + tstart + "')-678941) \ 108 AND mjd_obs <= (to_days('" + tend + "') - 678941) " 109 self.logger.infoPair("sql",sql) 104 110 try: 105 111 # XXX EAM : test output … … 212 218 Gets a camera-stage smf for this cam_id 213 219 ''' 214 def getCameraStageSmf(self, camID ):215 220 def getCameraStageSmf(self, camID, smfversion): 221 self.logger.infoPair("using", smfversion) 216 222 self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID)) 217 223 … … 237 243 # list all smf files if a neb path 238 244 files = [] 239 if path.startswith("neb"): 240 241 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? 242 f=os.popen("neb-ls -p "+path+"%smf") 243 for i in f.readlines(): 244 files.append(i.rstrip()) 245 246 # or not a neb path 247 else: 248 files = glob.glob(path + ".smf") 245 246 247 #there are a couple of states for smfversion 248 249 # "use_new" => we have reprocessed the data, use the .smf file 250 251 # "not_reproc" => this has not been reprocessed, use the .smf file 252 253 # "use_original" => we *may* have reprocessed the data, 254 # use (file).smf.original if camRun.state = 'full' and if there is a .smf and .orig 255 # use (file).smf if camRun.state = 'full' and if there is only a .smf 256 # fault out if camRun.state != 'full' -- that is a race condition we don't want. 257 258 259 if (smfversion == "not_reproc") or (smfversion == "use_new"): 260 if path.startswith("neb"): 261 262 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? 263 f=os.popen("neb-ls -p "+path+"%smf") 264 for i in f.readlines(): 265 files.append(i.rstrip()) 266 267 # or not a neb path 268 else: 269 files = glob.glob(path + ".smf") 270 271 if (smfversion == "use_original"): 272 # get state of camRun 273 sql = "SELECT state \ 274 FROM camProcessedExp \ 275 JOIN camRun USING(cam_id) \ 276 WHERE camRun.cam_id = %d" % camID 277 try: 278 rs = self.executeQuery(sql) 279 rs.first() 280 except: 281 self.logger.errorPair("Can't query for camRun.state for cam_id", str(camID)) 282 return None 283 284 try: 285 # get state of camRun.cam_id 286 camrunstate = rs.getString(1) 287 except: 288 self.logger.errorPair("No state found for cam_id", str(camID) ) 289 return None 290 291 if (camrunstate == "full"): 292 self.logger.infoPair("camRun.state is ",camrunstate) 293 # if full find smf files 294 if path.startswith("neb"): 295 296 # XXX EAM : this should just use the name (path.smf) instead of a wildcard, right? 297 fsmf=os.popen("neb-ls -p "+path+"%smf") 298 fsmfs = fsmf.readlines() 299 fsmforig=os.popen("neb-ls -p "+path+"%smf.original") 300 fsmforigs = fsmforig.readlines() 301 smfCount = len(fsmfs) 302 smfOrigCount = len (fsmforigs) 303 304 if (smfCount > 0 and smfOrigCount > 0): 305 #smf and original so use original 306 self.logger.infoPair("using:", path+"%smf.original") 307 for i in fsmforigs: 308 files.append(i.rstrip()) 309 310 if (smfCount > 0 and smfOrigCount == 0): 311 self.logger.infoPair("using:", path+"%smf") 312 #smf file and no smf.original use .smf 313 for i in fsmfs: 314 files.append(i.rstrip()) 315 316 #if not smf dont' find files ??) 317 # or not a neb path 318 319 else: 320 # this is broken for non neb - on the other hand we probably won't do .original if so 321 files = glob.glob(path + ".smf") 322 323 324 325 # if .smf and .original use .original 326 else: 327 self.logger.infoPair("camRun.state not full, can't ingest", str(camID) ) 328 return None 329 249 330 250 331 # XXX EAM : test output -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/ipptopspsdb.py
r35605 r36680 39 39 AND batch_type = '" + batchType + "' \ 40 40 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 41 AND comment = '" + self.skychunk.name + "' \ 42 AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ 41 43 AND merged = 1 \ 42 44 AND deleted_local = 0 \ … … 65 67 AND batch_type = '" + batchType + "' \ 66 68 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 69 AND comment = '" + self.skychunk.name + "' \ 70 AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ 67 71 AND purged = 1 \ 68 72 AND deleted_local = 0 \ … … 92 96 AND batch_type = '" + batchType + "' \ 93 97 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 98 AND comment = '" + self.skychunk.name + "' \ 99 AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ 94 100 AND purged = 1 \ 95 101 AND deleted_datastore = 0 \ … … 119 125 AND batch_type = '" + batchType + "' \ 120 126 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 121 AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ 127 AND comment = '" + self.skychunk.name + "' \ 128 AND datastore_product = '" + self.skychunk.datastoreProduct + "' \ 129 AND (merge_worthy = 1) \ 122 130 AND deleted_datastore = 0 AND block_delete_datastore = 0" 123 131 … … 144 152 AND batch_type = '" + batchType + "' \ 145 153 AND dvo_db = '" + self.skychunk.dvoLabel + "' \ 146 AND ( loaded_to_ODM = -1 ORmerge_worthy = 1) \154 AND (merge_worthy = 1) \ 147 155 AND deleted_dxlayer = 0 and block_delete_dxlayer = 0" 148 156 … … 1109 1117 ,queue_OB \ 1110 1118 ,parallel \ 1119 ,P2_smf_version \ 1120 ,ST_cmf_version \ 1121 ,trange_start \ 1122 ,trange_end \ 1111 1123 FROM skychunk \ 1112 1124 WHERE name = '" + self.skychunk.name + "'" … … 1155 1167 1156 1168 if rs.getInt(22) == 1: self.skychunk.parallel = True 1169 self.skychunk.P2_smf_version = rs.getString(23) 1170 self.skychunk.ST_cmf_version = rs.getString(24) 1171 self.skychunk.trange_start = rs.getString(25) 1172 self.skychunk.trange_end = rs.getString(26) 1157 1173 1158 1174 if self.skychunk.parallel: print "USING parallel" -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/odm.py
r35217 r36680 78 78 DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text 79 79 results['DETAILS'] = DETAILS 80 elif re.search("processing", STATE): 80 elif re.search("processing", STATE): 81 81 loadedToODM = 1 82 82 if re.search("MergeWorthy", MESSAGE): … … 87 87 mergeWorthy = 1 88 88 merged = 1 89 # if re.search("released", STATE): 90 # loadedToODM = 1 91 # mergeWorthy = 1 92 # merged = 1 93 # if re.search("released", MESSAGE): 94 # loadedToODM = 1 95 # mergeWorthy = 1 96 # merged = 1 89 97 except: 90 98 pass -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/queue.py
r35656 r36680 45 45 46 46 try: 47 print "why doesn't this work" 47 48 self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) 48 49 except: … … 71 72 for batchType in self.skychunk.batchTypes: 72 73 self.logger.infoPair("BatchType",batchType) 73 # 74 # 74 #self.logger.infoTitle("Previous failed datastore loads") 75 #self.publishAnyUnpublishedBatches(batchType) 75 76 76 77 # get totals for whole area to check if there is anything to do … … 98 99 self.skychunk.maxRa, 99 100 self.skychunk.minDec, 100 self.skychunk.maxDec) 101 self.skychunk.maxDec, 102 self.skychunk.trange_start, 103 self.skychunk.trange_end) 101 104 102 105 # EAM TEST I/O -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/jython/stackbatch.py
r35780 r36680 243 243 cnt=cnt+1 244 244 245 sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM"+psfCondition245 sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND "+psfCondition 246 246 self.scratchDb.execute(sql) 247 247 … … 303 303 modelLong = model; 304 304 305 #does it have the EXT_COVAR in there? 306 307 sql = "DESCRIBE SkyChip_xfit 'EXT_COVAR_%'" 308 rs = self.scratchDb.executeQuery(sql) 309 rs.first() 310 hasResult = rs.next() 311 if hasResult: 312 print "EXT_COVARs exist" 313 else: 314 print "No EXT_COVARS here" 315 316 ext_covar_sql = "" 317 318 if (hasResult): 319 ext_covar_sql = ", "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ 320 "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ 321 "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ 322 "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ 323 "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ 324 "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ 325 "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ 326 "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ 327 "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ 328 "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ 329 "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ 330 "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ 331 "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ 332 "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ 333 "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ 334 "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ 335 "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ 336 "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ 337 "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ 338 "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ 339 "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ 340 "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ 341 "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ 342 "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ 343 "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ 344 "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ 345 "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ 346 "+modelLong+"Covar77=b.EXT_COVAR_06_06" 347 348 349 350 305 351 sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ 306 352 "+model+"Radius=b.EXT_WIDTH_MAJ, \ … … 308 354 "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \ 309 355 "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ 310 "+model+"Phi=b.EXT_THETA, \ 311 "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ 312 "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ 313 "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ 314 "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ 315 "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ 316 "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ 317 "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ 318 "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ 319 "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ 320 "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ 321 "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ 322 "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ 323 "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ 324 "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ 325 "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ 326 "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ 327 "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ 328 "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ 329 "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ 330 "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ 331 "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ 332 "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ 333 "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ 334 "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ 335 "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ 336 "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ 337 "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ 338 "+modelLong+"Covar77=b.EXT_COVAR_06_06 \ 356 "+model+"Phi=b.EXT_THETA \ 357 "+ext_covar_sql+" \ 339 358 WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" 340 359 360 print sql 341 361 self.scratchDb.execute(sql) 342 362 343 363 # sersic fit has an extra parameter 344 if ippModelType == "PS_MODEL_SERSIC":364 if (ippModelType == "PS_MODEL_SERSIC" and hasResult > 0): 345 365 sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ 346 366 "+modelLong+"Covar18=b.EXT_COVAR_00_07, \ … … 633 653 634 654 # TODO temporarily loading 1st convolved fluxes into unconvolved fields 655 656 #This assumes that there are 3 different fluxs, the first is unconv, and the 2nd and 3rd are convolved. If there are more than 3 fluxes it will break. 657 # If there are more than 3 types of fluxes, we should do a count to find out how many there are and adjust this accordingly (make it more robust) 658 635 659 self.logger.infoPair("Adding fluxes", "un-convolved") 636 self.updateApFlxs("", " IS NULL")660 self.updateApFlxs("", "((b.table_index -1) % 3 = 0)") 637 661 self.logger.infoPair("Adding fluxes", "1st convolved") 638 self.updateApFlxs("c1", " < 7.0")662 self.updateApFlxs("c1", "((b.table_index -1) % 3 = 1)") 639 663 self.logger.infoPair("Adding fluxes", "2nd convolved") 640 self.updateApFlxs("c2", " > 7.0")664 self.updateApFlxs("c2", "((b.table_index -1) % 3 = 2)") 641 665 642 666 self.logger.infoPair("Adding", "petrosians for extended sources") … … 645 669 ,petRadiusErr=b.PETRO_RADIUS_ERR \ 646 670 ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \ 647 ,petFluxErr= ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736\671 ,petFluxErr=-999 \ 648 672 ,petR50=b.PETRO_RADIUS_50 \ 649 673 ,petR50Err=b.PETRO_RADIUS_50_ERR \ … … 653 677 WHERE a.ippDetectID=b.IPP_IDET" 654 678 self.scratchDb.execute(sql) 679 680 self.logger.infoPair("Adding", "petFluxErr for non nulls") 681 sql = "UPDATE StackApFlx as a, SkyChip_xsrc AS b SET \ 682 petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736 \ 683 WHERE a.ippDetectID=b.IPP_IDET \ 684 AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)" 685 # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc) 686 self.logger.infoPair("sql", sql) 687 self.scratchDb.execute(sql) 688 689 655 690 656 691 self.logger.infoPair("adding"," PrimaryF to stackApFlx") -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/Makefile.am
r35216 r36680 1 bin_SCRIPTS = checkOdmStatus.pl convertPhotCodesToXml.pl deleteFromDXLayer.pl pspsSchema2xml.pl 1 bin_SCRIPTS = checkOdmStatus.pl convertPhotCodesToXml.pl deleteFromDXLayer.pl pspsSchema2xml.pl deleteFromDXLayer2.pl 2 2 3 3 INSTALL = @INSTALL@ -C -
branches/eam_branches/ps2-tc3-20130727/ippToPsps/perl/deleteFromDXLayer.pl
r32930 r36680 18 18 ); 19 19 20 my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; 20 #my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; 21 22 #my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxwebservice.cgi"; 23 my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxlayer/dxwebservice.cgi"; 24 25 26 #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; 27 21 28 #my $dxWebServiceUrl = "http://166.122.172.121/cgi-bin/dxlayer/dxwebservice.cgi"; 22 29 my $ua = LWP::UserAgent->new;
Note:
See TracChangeset
for help on using the changeset viewer.
