Changeset 39565 for trunk/ippToPsps/jython/objectbatch.py
- Timestamp:
- May 1, 2016, 12:20:15 PM (10 years ago)
- File:
-
- 1 edited
-
trunk/ippToPsps/jython/objectbatch.py (modified) (8 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippToPsps/jython/objectbatch.py
r39255 r39565 147 147 # find Nsecfilt (or save in the db with dvopsps) 148 148 149 # the math below depends on filterCount = Nsecfilt and MeanObject.row being 1 counting but cps being 0 counting? 150 # cps.row has a count of MeanObject.row * Nsecfilt + Nfilter 151 # " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ 149 # the math below depends on filterCount = Nsecfilt and 150 # both MeanObject.row and cps.row being 1 counting, and 151 # filter[0] also being 1 counting 152 153 # cps.row - 1 = (cpt.row - 1)*Nsecfilt + (filter[0] - 1) 154 # cps.row = cpt.row * Nsecfilt - (Nsecfilt - filter[0]) 155 # cps.row = cpt.row * Nsecfilt - Nsecfilt + filter[0] 156 # [note that the sql below has parenthesis around (Nsecfilt - filter[0]) 152 157 153 158 sql = "UPDATE MeanObject JOIN \ … … 169 174 ,MeanObject." + filter[1] + "MeanApMagStd = MAG_AP_STDEV \ 170 175 ,MeanObject." + filter[1] + "MeanApMagNpt = NUSED_AP \ 171 ,MeanObject." + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) " 172 176 ,MeanObject." + filter[1] + "Flags = FLAGS " 173 177 174 178 try: self.scratchDb.execute(sql) … … 301 305 sqlLine.group("processingVersion", "'" + str(self.skychunk.processingVersion) + "'") 302 306 sqlLine.group("objInfoFlag", "FLAGS") 303 sqlLine.group("qualityFlag", "FLAGS >> 2 4& 0xFF")307 sqlLine.group("qualityFlag", "FLAGS >> 23 & 0xFF") 304 308 sqlLine.group("raStack", "RA_STK") 305 309 sqlLine.group("decStack", "DEC_STK") … … 339 343 self.updateObjectThinFromCps(cpsTableName) 340 344 341 # XXX EAM 20140724 : is this necessary?? 342 #objects can have out of range ra dec in dvo - need to find and kill them at the end 343 344 self.logger.infoPair("Determining", "ra/dec range") 345 346 raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) 347 raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) 348 decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) 349 decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) 350 351 self.logger.infoPair("R_MIN", raMin) 352 self.logger.infoPair("R_MAX", raMax) 353 self.logger.infoPair("D_MIN", decMin) 354 self.logger.infoPair("D_MAX", decMax) 355 #count out of range 356 357 sql = "SELECT count(*) FROM ObjectThin where \ 358 ObjectThin.decMean > " + str(decMax) + " \ 359 or ObjectThin.decMean < " + str(decMin) + " \ 360 or ObjectThin.raMean > " + str(raMax) + " \ 361 or ObjectThin.raMean < " + str(raMin) 362 363 rs = self.scratchDb.executeQuery(sql) 364 365 rs.first() 366 nToDelete = rs.getInt(1) 367 368 #delete out of range 369 370 371 sql = "DELETE FROM ObjectThin where \ 372 ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ 373 ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ 374 ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ 375 ObjectThin.raMean < (" + str(raMin) + " - .0033)" 376 self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") 377 378 try: 379 self.scratchDb.execute(sql) 380 except: 381 self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) 382 raise 383 384 ##Don't do this till after MeanObject 385 ##self.dvoObjects.purgeRegion(self.region) 345 if False: 346 # this chunk of code deletes objects which are out of ra,dec range for the table. 347 # this was a problem in an early version of DVO for cases where the astrometry went insane. 348 # this causes problems for the ra = 0,360 boundary (the test below does not handle that situation) 349 # and the restrictions below are poorly defined for the regions near the pole. 350 351 # in any case, ObjectThin needs to maintain the same order 352 # as the cpt table until MeanObjects have been created or 353 # the join to the cps table will fail 354 355 # XXX EAM 20140724 : is this necessary?? 356 # objects can have out of range ra dec in dvo - need to find and kill them at the end 357 358 self.logger.infoPair("Determining", "ra/dec range") 359 360 raMin = self.scratchDb.getFromdvoSkyTable("R_MIN",self.region) 361 raMax = self.scratchDb.getFromdvoSkyTable("R_MAX",self.region) 362 decMin = self.scratchDb.getFromdvoSkyTable("D_MIN",self.region) 363 decMax = self.scratchDb.getFromdvoSkyTable("D_MAX",self.region) 364 365 self.logger.infoPair("R_MIN", raMin) 366 self.logger.infoPair("R_MAX", raMax) 367 self.logger.infoPair("D_MIN", decMin) 368 self.logger.infoPair("D_MAX", decMax) 369 #count out of range 370 371 sql = "SELECT count(*) FROM ObjectThin where \ 372 ObjectThin.decMean > " + str(decMax) + " \ 373 or ObjectThin.decMean < " + str(decMin) + " \ 374 or ObjectThin.raMean > " + str(raMax) + " \ 375 or ObjectThin.raMean < " + str(raMin) 376 377 rs = self.scratchDb.executeQuery(sql) 378 379 rs.first() 380 nToDelete = rs.getInt(1) 381 382 #delete out of range 383 384 385 sql = "DELETE FROM ObjectThin where \ 386 ObjectThin.decMean > (" + str(decMax) + " + .0033) or \ 387 ObjectThin.decMean < (" + str(decMin) + " - .0033) or \ 388 ObjectThin.raMean > (" + str(raMax) + " + .0033) or \ 389 ObjectThin.raMean < (" + str(raMin) + " - .0033)" 390 self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") 391 392 try: 393 self.scratchDb.execute(sql) 394 except: 395 self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) 396 raise 397 386 398 self.logger.infoPair("updatePspsUniqueIDs","start") 387 399 self.updatePspsUniqueIDs() 388 400 self.logger.infoPair("updatePspsUniqueIDs","end") 401 389 402 self.logger.infoPair("Dropping row column from", "ObjectThin table") 390 403 self.scratchDb.dropColumn("ObjectThin", "row") 391 ##self.logger.infoPair("Purging from scratch Db", self.region + " region")392 404 self.logger.infoPair("Dropped row column", "objectThin") 393 ##Don't do this till after MeanObject394 405 395 406 self.setMinMaxObjID(["ObjectThin"]) … … 415 426 self.scratchDb.addRowCountColumn("MeanObject", "row") 416 427 417 418 419 420 421 ##self.scratchDb.addRowCountColumn(cpsTableName, "row")422 428 self.logger.infoPair("update MeanObjects from ","cps table") 423 429 self.updateMeanObjectFromCps(cpsTableName) … … 434 440 435 441 if not self.populateObjectThinTable(): return False 436 #if not self.populateObjectCalColorTable(): return False437 442 if not self.populateMeanObjectTable(): return False 438 443 439 444 # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number 445 446 ### XXX the code below first removes duplicate objID entries 447 ### from ObjectThin, then does the same for MeanObject. This 448 ### is a big problem: we have no guarantee that the surviving 449 ### rows are the correct matched rows. 450 451 ### force objID uniqueness on *** ObjectThin *** 440 452 self.logger.infoPair("Forcing uniqueness on", "objID in ObjectThin table") 441 453 rowCountBefore = self.scratchDb.getRowCount("ObjectThin") … … 457 469 self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) 458 470 471 ### force objID uniqueness on *** MeanObject *** 459 472 self.logger.infoPair("Forcing uniqueness on", "objID in MeanObject table") 460 473 rowCountBefore = self.scratchDb.getRowCount("MeanObject") … … 476 489 self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) 477 490 491 # delete the cpt, cps tables from the scratch mysql (or we will run out of space) 478 492 self.dvoObjects.purgeRegion(self.region) 479 493 480 #this is abuse of something but this is how I get the object batches to crash to further investigate them 481 482 # rowCountAfter = self.scratchDb.getRowCount("Object") 483 return True 484 # return False 494 return True
Note:
See TracChangeset
for help on using the changeset viewer.
