Index: trunk/ippToPsps/jython/detectionbatch.py =================================================================== --- trunk/ippToPsps/jython/detectionbatch.py (revision 35564) +++ trunk/ippToPsps/jython/detectionbatch.py (revision 35605) @@ -450,5 +450,5 @@ ,MOMENTS_XY \ ,MOMENTS_YY \ - ,AP_FLUX / " + self.header['EXPTIME'] + " \ + ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ ,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \ @@ -469,5 +469,7 @@ # ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + " # ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736) -# ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " + +# I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX +# AP_FLUX is no ,AP_FLUX / " + self.header['EXPTIME'] + " self.scratchDb.execute(sql) @@ -1035,5 +1037,8 @@ # XXX EAM NOTE : this is fragile : requires PS1_V4 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" + # columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" + + # XXX EAM NOTE : this is fragile : requires PS1_V4 + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" return super(DetectionBatch, self).importIppTables(columns, regex) Index: trunk/ippToPsps/jython/dvo.py =================================================================== --- trunk/ippToPsps/jython/dvo.py (revision 35564) +++ trunk/ippToPsps/jython/dvo.py (revision 35605) @@ -65,12 +65,11 @@ if not self.correctDvo: print "*******************************************************************************" - self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.skychunk.dvoLocation + "'?") - response = raw_input("(y/n) ") - if response == "y": - self.logger.warning("Are you ABSOLUTELY sure you want to do this?") - response = raw_input("(y/n) ") - if response == "y": - self.resetAllTables() - + self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation) + # response = raw_input("(y/n) ") + # if response == "y": + # self.logger.warning("Are you ABSOLUTELY sure you want to do this?") + # response = raw_input("(y/n) ") + # if response == "y": + self.resetAllTables() ''' @@ -597,13 +596,14 @@ self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost) - ## print "*******************************************************************************" - ## self.logger.warning("about to delete detection table and re-ingest; is this correct?") - ## response = raw_input("(y/n) ") - ## if response != "y": - ## raise - ## self.logger.warning("Are you ABSOLUTELY sure you want to do this?") - ## response = raw_input("(y/n) ") - ## if response != "y": - ## raise + if True: + print "*******************************************************************************" + self.logger.warning("about to delete detection table and re-ingest; is this correct?") + #response = raw_input("(y/n) ") + #if response != "y": + # raise + #self.logger.warning("Are you ABSOLUTELY sure you want to do this?") + #response = raw_input("(y/n) ") + #if response != "y": + # raise # blow away existing dvoDetection table & re-crate Index: trunk/ippToPsps/jython/ippjython =================================================================== --- trunk/ippToPsps/jython/ippjython (revision 35564) +++ trunk/ippToPsps/jython/ippjython (revision 35605) @@ -19,4 +19,5 @@ set JYTHON = $JARDIR/jython2.5.3b1/jython.jar # set JYTHON = ~/jython2.5.3b1/jython.jar +#set JYTHON = $JARDIR/jython.jar if (! -r $JYTHON) then Index: trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- trunk/ippToPsps/jython/ipptopspsdb.py (revision 35564) +++ trunk/ippToPsps/jython/ipptopspsdb.py (revision 35605) @@ -785,5 +785,6 @@ survey, \ dvo_db, \ - datastore_product \ + datastore_product, \ + comment \ ) VALUES ( \ '" + batchType + "', \ @@ -791,5 +792,6 @@ '" + self.skychunk.survey + "', \ '" + self.skychunk.dvoLabel + "', \ - '" + self.skychunk.datastoreProduct + "' \ + '" + self.skychunk.datastoreProduct + "', \ + '" + self.skychunk.name + "' \ )" self.logger.infoPair("heather:","sql") @@ -1215,4 +1217,6 @@ ('" + self.skychunk.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.skychunk.boxSize) + ")" + # print "insertBox: ", sql + self.execute(sql) id = self.getBoxId(ra, dec) @@ -1556,6 +1560,6 @@ sql = "SELECT id, ra_center, dec_center \ - FROM box \ - WHERE skychunk = '" + self.skychunk.name + "' \ + FROM box join skychunk on box.skychunk = skychunk.name \ + WHERE dvo_label = '" + self.skychunk.dvoLabel + "' \ ORDER BY ra_center, dec_center" @@ -1599,4 +1603,5 @@ sql = "INSERT INTO all_pending (stage_id, ra_bore, dec_bore) \ VALUES (%d, %f, %f)" % (row[0], row[1], row[2]) + # print "sql 1: ", sql self.execute(sql) except: continue @@ -1604,4 +1609,21 @@ count = self.getRowCount("all_pending") self.logger.infoPair("Items written to Db", "%d" % count) + + # try: + # self.execute("DROP TABLE all_pending_alt") + # except: + # pass + # + # self.execute("CREATE TABLE all_pending_alt (stage_id bigint(20), ra_bore float, dec_bore float)") + # for row in rows: + # try: + # sql = "INSERT INTO all_pending_alt (stage_id, ra_bore, dec_bore) \ + # VALUES (%d, %f, %f)" % (row[0], row[1], row[2]) + # print "sql 2: ", sql + # self.execute(sql) + # except: continue + # + # count = self.getRowCount("all_pending_alt") + # self.logger.infoPair("Items written to alt Db", "%d" % count) ''' @@ -1624,4 +1646,6 @@ AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC) + print "items: ", sql + try: rs = self.executeQuery(sql) Index: trunk/ippToPsps/jython/queue.py =================================================================== --- trunk/ippToPsps/jython/queue.py (revision 35564) +++ trunk/ippToPsps/jython/queue.py (revision 35605) @@ -37,10 +37,10 @@ self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) - # print "made datastore" + print "made datastore" # connect to scratch database self.scratchDb = ScratchDb(self.logger, self.config) - # print "connect to scratchdb" + print "connect to scratchdb" try: @@ -50,5 +50,5 @@ raise - # print "got objects" + print "got objects" self.skychunk.printAll() @@ -62,13 +62,13 @@ ''' def run(self): - + print "here" # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time) while True: self.ippToPspsDb.removeAllBoxes() - + print "remove boxes" # queue up batches that are processed but not loaded to datastore for batchType in self.skychunk.batchTypes: - + self.logger.infoPair("BatchType",batchType) # self.logger.infoTitle("Previous failed datastore loads") # self.publishAnyUnpublishedBatches(batchType) @@ -76,6 +76,7 @@ # get totals for whole area to check if there is anything to do processedIDs = self.ippToPspsDb.getProcessedIDs(batchType) + print "got processed ids" consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType) - + print "got here" # for object batches, get full list of stuff from Dvo # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo @@ -136,16 +137,17 @@ if (dec > 85): raise - # print "dec, dD: ", dec, dD + print "dec, dD: ", dec, dD ,self.skychunk.maxDec # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds - dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec)) + #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec)) + dR = 0.5*self.skychunk.boxSize ra = self.skychunk.minRa + dR - # print "ra, dR: ", ra, dR + print "ra, dR, maxRa", ra, dR, self.skychunk.maxRa # XXX not sure why this is not done with a for-loop... while dec <= self.skychunk.maxDec: while ra <= self.skychunk.maxRa: - + # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds minRA = ra - dR @@ -175,5 +177,6 @@ if (dec > 85): raise - dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec)) + #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec)) + dR = 0.5*self.skychunk.boxSize ra = self.skychunk.minRa + dR Index: trunk/ippToPsps/jython/stackbatch.py =================================================================== --- trunk/ippToPsps/jython/stackbatch.py (revision 35564) +++ trunk/ippToPsps/jython/stackbatch.py (revision 35605) @@ -504,5 +504,5 @@ ,MOMENTS_R1 \ ,MOMENTS_RH \ - ,AP_FLUX / " + str(self.expTime) + " \ + ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \ ,NULL \ ,KRON_FLUX / " + str(self.expTime) + " \ @@ -519,4 +519,7 @@ # ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) # ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " + +# I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX +# ,AP_FLUX / " + str(self.expTime) + " # print "sql: ", sql