Index: trunk/ippToPsps/doc/boxes.txt =================================================================== --- trunk/ippToPsps/doc/boxes.txt (revision 35203) +++ trunk/ippToPsps/doc/boxes.txt (revision 35203) @@ -0,0 +1,19 @@ + +** problem 1 : boxes are fixed sizes in RA, but need to vary in width *at least* based on DEC +** problem 2 : dvopsps needs to grab a region with a border of width 1.65deg in projection (varying for RA, fixed for DEC) + * this can be dealt with in dvo.py + +boxes are defined in queue.py:124 + * insertBox + +(loop is over ra,dec explicitly in half-box-size steps) + +dvo.py uses boxes for region, but could be modified easily + +the 'setSkyArea' functions use the box sizes + +** recommended changes: + 1) dvo nativeIngestDetections should either take explicit dR,dD arguments or a size without borders + +skychunk.boxSizeWithBorder should be dropped (not actually used anyway) + Index: trunk/ippToPsps/doc/dvopsps.txt =================================================================== --- trunk/ippToPsps/doc/dvopsps.txt (revision 35203) +++ trunk/ippToPsps/doc/dvopsps.txt (revision 35203) @@ -0,0 +1,19 @@ + +The program called dvopsps inserts the detection / object data needed +to assign the IDs and some other data to the smf/cmf files. the +fields it inserts are: + +imageID +detID +measure.extID (psps detect ID) : detection +ippObjID (catID*1e9 + objID) : detection +average.extID (psps object ID) : detection +dbflags : detection +zp : detection calib +zpErr : detection calib +airmass : detection calib +exptime : detection calib +ra : detection calib +dec : detection calib +raErr : detection calib +decErr : detection calib Index: trunk/ippToPsps/jython/dvo.py =================================================================== --- trunk/ippToPsps/jython/dvo.py (revision 35200) +++ trunk/ippToPsps/jython/dvo.py (revision 35203) @@ -211,4 +211,7 @@ def setSkyAreaAsStripe(self, ra=0., width=4.): + print "this function should not reached now, error" + raise + self.setSkyArea(ra, ra+width, -91., 91.) @@ -217,4 +220,7 @@ ''' def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0): + + print "this function should not reached now, error" + raise halfSide = side/2.0 @@ -223,4 +229,6 @@ ''' Determines what DVO files need to be ingested to the defines region + XXX EAM : this function has been substantially reduced / stripped down to work with the parallel dvo + XXX EAM : some additional review of the goals would be useful ''' def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91): @@ -584,5 +592,5 @@ includes purging detections outside sky area ''' - def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize): + def nativeIngestDetections(self, boxId, boxDim): # XXX put the chunk below in a separate method @@ -628,7 +636,16 @@ self.loadImages() - # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx - halfSize = boxSize / 2.0 - # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize + # the box dimensions are the area used to select the items of + # interest from the gpc1 database (camera smfs / skycal cmfs). + # for dvopsps, we need to include a border region large enough + # to grab all detections / objects which are contributed by an + # item with the center in the given box + dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC'])) + dD_border = self.skychunk.BORDER + + minRA = boxDim['minRA'] - dR_border + maxRA = boxDim['maxRA'] + dR_border + minDEC = boxDim['minDEC'] - dD_border + maxDEC = boxDim['maxDEC'] + dD_border # TODO path to DVO prog hardcoded temporarily @@ -640,8 +657,8 @@ cmd += " -D CATDIR " + self.skychunk.dvoLocation cmd += " -region " - cmd += " " + str(raCenter-halfSize) - cmd += " " + str(raCenter+halfSize) - cmd += " " + str(decCenter-halfSize) - cmd += " " + str(decCenter+halfSize) + cmd += " " + str(minRA) + cmd += " " + str(maxRA) + cmd += " " + str(minDEC) + cmd += " " + str(maxDEC) if self.skychunk.parallel: Index: trunk/ippToPsps/jython/ipptopspsdb.py =================================================================== --- trunk/ippToPsps/jython/ipptopspsdb.py (revision 35200) +++ trunk/ippToPsps/jython/ipptopspsdb.py (revision 35203) @@ -4,4 +4,5 @@ import sys import os +import math import logging @@ -997,6 +998,6 @@ self.skychunk.boxSize = rs.getDouble(10) - self.skychunk.halfBox = self.skychunk.boxSize/2.0 - self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) + # self.skychunk.halfBox = self.skychunk.boxSize/2.0 + # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) self.skychunk.basePath = rs.getString(11) @@ -1031,4 +1032,6 @@ ''' Gets id of box with these coords + XXX EAM : this code implies we can have multiple boxes at the same location for the same + XXX EAM : skychunk distinguished by having different sizes. that seems quite bad ''' def getBoxId(self, ra, dec): @@ -1037,6 +1040,7 @@ WHERE skychunk = '" + self.skychunk.name + "' \ AND ra_center = " + str(ra) + " \ - AND dec_center = " + str(dec) + " \ - AND box_side = " + str(self.skychunk.boxSize) + AND dec_center = " + str(dec) + + # AND box_side = " + str(self.skychunk.boxSize) id = -1 @@ -1325,4 +1329,18 @@ boxDim['SIDE'] = rs.getFloat(3) + # SIDE is the linear full-width of the box + + # XXX EAM : need to do something special at the poles... + if (boxDim['DEC'] > 85): raise + + # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds + dR = 0.5*boxDim['SIDE'] / math.cos(math.radians(boxDim['DEC'])) + dD = 0.5*boxDim['SIDE'] + + boxDim['minRA'] = boxDim['RA'] - dR + boxDim['maxRA'] = boxDim['RA'] + dR + boxDim['minDEC'] = boxDim['DEC'] - dD + boxDim['maxDEC'] = boxDim['DEC'] + dD + return boxDim Index: trunk/ippToPsps/jython/loader.py =================================================================== --- trunk/ippToPsps/jython/loader.py (revision 35200) +++ trunk/ippToPsps/jython/loader.py (revision 35203) @@ -157,10 +157,7 @@ self.logger.infoSeparator() self.logger.infoTitle("New box") - self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'])) + self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f -- sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC'])) self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids)) - boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2 / math.cos(math.radians(boxDim['DEC']))) - boxSizeSansBorder = boxDim['SIDE'] - boxSize = boxSizeWithBorder - self.logger.infoPair("using box of size", str(boxSize)) + useFullTables = 0 if batchType != "OB": @@ -169,4 +166,7 @@ # look in DVO for this box (with extra border) self.logger.infoPair("Querying DVO for this sky area", "") + + # XXXX EAM : This code should not be used anymore + boxSize = 0.0 self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSize) #self.dvoDetections.setSkyArea() @@ -189,12 +189,14 @@ useFullTables = 1 else: - # XXX EAM : this is not currently an - # option. either remove the non-native - # ingest code or make it an option - # if we are using the native loader, always use it - # need to work out a good boundary / region strategy in coordination with - # impact of mysql insertion + # XXX EAM : this is not currently optional + # (it is the only path). either remove the + # non-native ingest code or make it an + # option if we are using the native + # loader, always use it need to work out a + # good boundary / region strategy in + # coordination with impact of mysql + # insertion if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost): - self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSize) + self.dvoDetections.nativeIngestDetections(boxId, boxDim) useFullTables = 1 Index: trunk/ippToPsps/jython/queue.py =================================================================== --- trunk/ippToPsps/jython/queue.py (revision 35200) +++ trunk/ippToPsps/jython/queue.py (revision 35203) @@ -116,12 +116,29 @@ # starting positions - ra = self.skychunk.minRa + self.skychunk.halfBox - dec = self.skychunk.minDec + self.skychunk.halfBox - - while ra <= self.skychunk.maxRa: - while dec <= self.skychunk.maxDec: + + dD = 0.5*skychunk.boxSize + dec = self.skychunk.minDec + dD + + # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds + dR = 0.5*skychunk.boxSize / math.cos(math.radians(dec)) + ra = self.skychunk.minRa + dR + + while dec <= self.skychunk.maxDec: + + # XXX EAM : need to do something special at the poles... + if (dec > 85): raise + + while ra <= self.skychunk.maxRa: + # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds + minRA = ra - dR + maxRA = ra + dR + minDEC = dec - dD + maxDEC = dec + dD + + XXX EAM: box_id = self.ippToPspsDb.insertBox(ra, dec) + XXX EAM: ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec) @@ -133,9 +150,11 @@ if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids) + ra = ra + 2*dR - dec = dec + self.skychunk.boxSize - dec = self.skychunk.minDec + self.skychunk.halfBox - ra = ra + self.skychunk.boxSize - + dec = dec + 2*dD + + dR = 0.5*skychunk.boxSize / math.cos(math.radians(dec)) + ra = self.skychunk.minRa + dR + self.logger.info("+-------------+-------------+-------------+-------------+")