VOTable description of PSPS table StackType
@@ -788,7 +662,7 @@
| RADIAL_FLUX | 134217728 | radial flux measurements calculated |
| SIZE_SKIPPED | 268435456 | size could not be determined |
- | ON_SPIKE | 536870912 | peak lands on diffraction spike |
- | ON_GHOST | 1073741824 | peak lands on ghost or glint |
- | OFF_CHIP | 2147483648 | peak lands off edge of chip |
+ | PEAK_ON_SPIKE | 536870912 | peak lands on diffraction spike |
+ | PEAK_ON_GHOST | 1073741824 | peak lands on ghost or glint |
+ | PEAK_OFF_CHIP | 2147483648 | peak lands off edge of chip |
| DIFF_WITH_SINGLE | 4294967296 | diff source matched to a single positive detection |
| DIFF_WITH_DOUBLE | 8589934592 | diff source matched to positive detections in both images |
@@ -868,7 +742,7 @@
| RADIAL_FLUX | 134217728 | radial flux measurements calculated |
| SIZE_SKIPPED | 268435456 | size could not be determined |
- | ON_SPIKE | 536870912 | peak lands on diffraction spike |
- | ON_GHOST | 1073741824 | peak lands on ghost or glint |
- | OFF_CHIP | 2147483648 | peak lands off edge of chip |
+ | PEAK_ON_SPIKE | 536870912 | peak lands on diffraction spike |
+ | PEAK_ON_GHOST | 1073741824 | peak lands on ghost or glint |
+ | PEAK_OFF_CHIP | 2147483648 | peak lands off edge of chip |
| DIFF_WITH_SINGLE | 4294967296 | diff source matched to a single positive detection |
| DIFF_WITH_DOUBLE | 8589934592 | diff source matched to positive detections in both images |
Index: branches/eam_branches/ipp-20130207/ippToPsps/config/tables.OB.vot
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/config/tables.OB.vot (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/config/tables.OB.vot (revision 35235)
@@ -300,6 +300,6 @@
Data release
-
- MJD of last update
+
+ batch id
Index: branches/eam_branches/ipp-20130207/ippToPsps/config/tables.P2.vot
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/config/tables.P2.vot (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/config/tables.P2.vot (revision 35235)
@@ -147,4 +147,7 @@
calibration modification number
+
+ batchID
+
Data release
Index: branches/eam_branches/ipp-20130207/ippToPsps/config/tables.ST.vot
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/config/tables.ST.vot (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/config/tables.ST.vot (revision 35235)
@@ -105,4 +105,7 @@
calibration modification number
+
+ batchID
+
Data release
Index: branches/eam_branches/ipp-20130207/ippToPsps/configure.ac
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/configure.ac (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/configure.ac (revision 35235)
@@ -49,4 +49,5 @@
jars/Makefile
jython/Makefile
+ perl/Makefile
src/Makefile
Doxyfile
Index: branches/eam_branches/ipp-20130207/ippToPsps/doc/boxes.txt
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/doc/boxes.txt (revision 35235)
+++ branches/eam_branches/ipp-20130207/ippToPsps/doc/boxes.txt (revision 35235)
@@ -0,0 +1,19 @@
+
+** problem 1 : boxes are fixed sizes in RA, but need to vary in width *at least* based on DEC
+** problem 2 : dvopsps needs to grab a region with a border of width 1.65deg in projection (varying for RA, fixed for DEC)
+ * this can be dealt with in dvo.py
+
+boxes are defined in queue.py:124
+ * insertBox
+
+(loop is over ra,dec explicitly in half-box-size steps)
+
+dvo.py uses boxes for region, but could be modified easily
+
+the 'setSkyArea' functions use the box sizes
+
+** recommended changes:
+ 1) dvo nativeIngestDetections should either take explicit dR,dD arguments or a size without borders
+
+skychunk.boxSizeWithBorder should be dropped (not actually used anyway)
+
Index: branches/eam_branches/ipp-20130207/ippToPsps/doc/dvopsps.txt
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/doc/dvopsps.txt (revision 35235)
+++ branches/eam_branches/ipp-20130207/ippToPsps/doc/dvopsps.txt (revision 35235)
@@ -0,0 +1,19 @@
+
+The program called dvopsps inserts the detection / object data needed
+to assign the IDs and some other data to the smf/cmf files. the
+fields it inserts are:
+
+imageID
+detID
+measure.extID (psps detect ID) : detection
+ippObjID (catID*1e9 + objID) : detection
+average.extID (psps object ID) : detection
+dbflags : detection
+zp : detection calib
+zpErr : detection calib
+airmass : detection calib
+exptime : detection calib
+ra : detection calib
+dec : detection calib
+raErr : detection calib
+decErr : detection calib
Index: branches/eam_branches/ipp-20130207/ippToPsps/doc/notes.txt
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/doc/notes.txt (revision 35235)
+++ branches/eam_branches/ipp-20130207/ippToPsps/doc/notes.txt (revision 35235)
@@ -0,0 +1,12 @@
+
+example of joining ipptopsps and gpc1 tables:
+
+mysql -h ipp006 -u dvo -p ipptopsps -e "select batch_id, stage_id from batch where batch_type = 'P2'" > P2.batches.dat
+
+on ippdb01: cp P2.batches.dat /tmp/datadump/
+
+mysql -h ippdb01 -u dvo -p gpc1
+mysql> create temporary table eam_p2_batches (batch_id int, cam_id int);
+mysql> create temporary table eam_st_batches (batch_id int, stack_id int);
+mysql> load data infile '/tmp/datadump/P2.batches.dat' into table eam_p2_batches ignore 1 lines (batch_id, cam_id);
+mysql> select degrees(ra) as ra_deg, degrees(decl) as dec_deg, camRun.cam_id, batch_id, chip_id, exp_id, exp_name from camRun join chipRun using (chip_id) join rawExp using (exp_id) join eam_p2_batches on (eam_p2_batches.cam_id = camRun.cam_id) where batch_id is not null and degrees(ra) > 298.0 limit 100;
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/batch.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/batch.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/batch.py (revision 35235)
@@ -544,4 +544,9 @@
# for OB, this imports object data from DVO
if not self.importIppTables():
+ if self.batchType == "OB":
+ self.logger.errorPair("skipping this batch", "no dvo tables for this region")
+ self.ippToPspsDb.updateProcessed(self.batchID, 1)
+ return True
+
self.logger.errorPair("Aborting this batch", "could not import IPP tables")
self.ippToPspsDb.updateProcessed(self.batchID, -1)
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/config.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/config.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/config.py (revision 35235)
@@ -38,4 +38,5 @@
## output file. we blindly set name == argv[1]
## programs which are called without argv[1] will use 'none' in the log file
+ self.name = None
if len(sys.argv) >= 2:
self.name = sys.argv[1]
@@ -57,4 +58,7 @@
self.settingsDoc = ElementTree(file=self.settingsPath)
self.logPath = self.settingsDoc.find("logPath").text
+ self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text
+
+
print "config.programName: ", self.programName
@@ -63,4 +67,14 @@
print "config.logPath: ", self.logPath
print "config.test: ", self.test
+ print "config.czarPlotsPath: ", self.czarPlotsPath
+
+ root = self.settingsDoc.getroot()
+ localdb = root.find('localdatabase')
+ for loaderhost in localdb.findall('loaderhost'):
+ name = loaderhost.get('name')
+ dbname = loaderhost.find('dbname').text
+ dbhost = loaderhost.find('host').text
+ print "loaderhost / dbhost : ", name, dbname, dbhost
+
'''
@@ -69,14 +83,10 @@
def getLogger(self, host, pid, stdout=1, sendToFile=0):
- print "get Logger: ", host, pid
logging.setLoggerClass(PSLogger)
- print "done Logger 1"
self.logger = logging.getLogger(self.programName)
- print "done Logger 2"
if (self.name is None):
self.logger.setup(self.programName, self.logPath, "none", host, pid, stdout, sendToFile)
else:
self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile)
- print "done Logger 3"
return self.logger
@@ -85,6 +95,49 @@
Various db metadata getters
'''
- def getDbName(self, dbType): return self.settingsDoc.find(dbType +"/name").text
- def getDbHost(self, dbType): return self.settingsDoc.find(dbType +"/host").text
- def getDbUser(self, dbType): return self.settingsDoc.find(dbType +"/user").text
- def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text
+ def getDbName(self, dbType):
+ if dbType == 'localdatabase' or dbType == 'localdatabase_test':
+ return self.scratchDbLeaf.find('name').text
+ else:
+ return self.settingsDoc.find(dbType +"/name").text
+
+ def getDbHost(self, dbType):
+ if dbType == 'localdatabase' or dbType == 'localdatabase_test':
+ return self.scratchDbLeaf.find('host').text
+ else:
+ return self.settingsDoc.find(dbType +"/host").text
+
+ def getDbUser(self, dbType):
+ if dbType == 'localdatabase' or dbType == 'localdatabase_test':
+ return self.scratchDbLeaf.find('user').text
+ else:
+ return self.settingsDoc.find(dbType +"/user").text
+
+ def getDbPassword(self, dbType):
+ if dbType == 'localdatabase' or dbType == 'localdatabase_test':
+ return self.scratchDbLeaf.find('password').text
+ else:
+ return self.settingsDoc.find(dbType +"/password").text
+
+ def setScratchDbLeaf(self, myhost):
+ root = self.settingsDoc.getroot()
+ if (self.test):
+ localdb = root.find('localdatabase_test')
+ else:
+ localdb = root.find('localdatabase')
+
+ for loaderhost in localdb.findall('loader'):
+ name = loaderhost.get('mach')
+ print "name, myhost: ", name, myhost
+ if name == myhost:
+ self.scratchDbLeaf = loaderhost
+ return 1
+
+ for loaderhost in localdb.findall('loader'):
+ name = loaderhost.get('mach')
+ print "name, myhost: ", name, myhost
+ if name == 'generic':
+ self.scratchDbLeaf = loaderhost
+ return 1
+
+ print "Config Error: cannot find valid scratch db matching ", myhost, "or generic"
+ raise
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/detectionbatch.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/detectionbatch.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/detectionbatch.py (revision 35235)
@@ -215,5 +215,5 @@
self.scratchDb.execute(sql)
-
+ self.scratchDb.updateAllRows("FrameMeta", "batchID", str(self.batchID))
self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
self.scratchDb.updateFilterID("FrameMeta", self.filter)
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/dvo.py (revision 35235)
@@ -6,4 +6,5 @@
import sys
import os
+import math
import logging
import glob
@@ -33,5 +34,5 @@
'''
- def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
+ def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
# set up logging
@@ -40,4 +41,5 @@
self.skychunk = skychunk
self.ippToPspsDb = ippToPspsDb
+ self.scratchDb = scratchDb
# set up empty lists
@@ -48,9 +50,4 @@
self.regionsIngestedButOutOfDate = []
- # connect to the specified scratch database, if a name is given, otherwise use the first version
- try:
- self.scratchDb = ScratchDb(self.logger, self.config, '1', scratchDbName)
- except: raise
-
# we have (still) 3 modes of ingest from dvo cpt/cps/cpm files
# 1) dvograbber (useFullTables == FALSE),
@@ -211,4 +208,7 @@
def setSkyAreaAsStripe(self, ra=0., width=4.):
+ print "this function should not reached now, error"
+ raise
+
self.setSkyArea(ra, ra+width, -91., 91.)
@@ -217,4 +217,7 @@
'''
def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0):
+
+ print "this function should not reached now, error"
+ raise
halfSide = side/2.0
@@ -223,4 +226,6 @@
'''
Determines what DVO files need to be ingested to the defines region
+ XXX EAM : this function has been substantially reduced / stripped down to work with the parallel dvo
+ XXX EAM : some additional review of the goals would be useful
'''
def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91):
@@ -263,16 +268,18 @@
# get paths for this region for the file types we are interested in
- paths = []
- for fileType in self.ingestFileTypes:
- paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
+ # XXX ipptopsps cannot check files -- it does not know where they are
+ ## paths = []
+ ## for fileType in self.ingestFileTypes:
+ ## paths.append(self.skychunk.dvoLocation + "/" + regionPath + "." + fileType)
# check for the existence of all interested file types
- skipThisRegion = False
- for path in paths:
- if not os.path.isfile(path):
- skipThisRegion = True
- break
-
- if skipThisRegion: continue
+ # XXX ipptopsps cannot check files -- it does not know where they are
+ ## skipThisRegion = False
+ ## for path in paths:
+ ## if not os.path.isfile(path):
+ ## skipThisRegion = True
+ ## break
+
+ ## if skipThisRegion: continue
# ok, have real, populated files so add this region to the list
@@ -281,10 +288,11 @@
# if we have already imported up-to-date versions of all interested files, then we skip this region
- alreadyIngested = True
- for path in paths:
- if not self.scratchDb.alreadyImportedThisDvoTable(path):
- alreadyIngested = False
- break
-
+ ## alreadyIngested = True
+ ## for path in paths:
+ ## if not self.scratchDb.alreadyImportedThisDvoTable(path):
+ ## alreadyIngested = False
+ ## break
+
+ alreadyIngested = False
if alreadyIngested:
self.regionsAlreadyIngested.append(regionPath)
@@ -299,11 +307,13 @@
# check if we have out-of-date versions of any interested files, if so, add region to the purge list
- outOfDate = False
- for path in paths:
- if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
- outOfDate = True
- break
-
- if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+ ## outOfDate = False
+ ## for path in paths:
+ ## if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path):
+ ## outOfDate = True
+ ## break
+ ##
+ ## if outOfDate: self.regionsIngestedButOutOfDate.append(regionPath)
+
+ print "using region: ", regionPath
self.regionsToIngest.append(regionPath)
@@ -389,6 +399,8 @@
self.logger.infoPair("Purging", "%d regions" % len(regions))
- detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
- if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
+ self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
+
+ ## detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(regions, self.ingestFileTypes)
+ ## if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount)
return True
@@ -420,8 +432,11 @@
size = 0.0
+ return size
+
for region in regions:
self.logger.infoPair("sizes for region", region)
# get combined size of all interested files
for fileType in self.ingestFileTypes:
+ print "trying ", fileType
size = size + self.getDiskSize(self.skychunk.dvoLocation + "/" + region + "." + fileType)
# EAM TEST I/O
@@ -574,12 +589,22 @@
includes purging detections outside sky area
'''
- def nativeIngestDetections(self, boxId, raCenter, decCenter, boxSize):
+ def nativeIngestDetections(self, boxId, boxDim):
# XXX put the chunk below in a separate method
# blow away existing dvoDetection table
- # clear the 'ingested' field for all boxes
- self.ippToPspsDb.clearIngestedBoxes()
+ # clear the 'ingested' field for all boxes owned by this host
+ self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
+ ## print "*******************************************************************************"
+ ## self.logger.warning("about to delete detection table and re-ingest; is this correct?")
+ ## response = raw_input("(y/n) ")
+ ## if response != "y":
+ ## raise
+ ## self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
+ ## response = raw_input("(y/n) ")
+ ## if response != "y":
+ ## raise
+
# drop detections table
self.logger.infoPair("Dropping table", self.scratchDb.dvoDetectionTable)
@@ -608,7 +633,16 @@
self.loadImages()
- # dvopsps -D catdir CATDIR -region .... -dbhost xx -dbname xx -dbuser xx -dbpass xx
- halfSize = boxSize / 2.0
- # -region raCenter-halfSize raCenter+halfSize decCenter-halfSize decCenter+halfSize
+ # the box dimensions are the area used to select the items of
+ # interest from the gpc1 database (camera smfs / skycal cmfs).
+ # for dvopsps, we need to include a border region large enough
+ # to grab all detections / objects which are contributed by an
+ # item with the center in the given box
+ dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC']))
+ dD_border = self.skychunk.BORDER
+
+ minRA = boxDim['minRA'] - dR_border
+ maxRA = boxDim['maxRA'] + dR_border
+ minDEC = boxDim['minDEC'] - dD_border
+ maxDEC = boxDim['maxDEC'] + dD_border
# TODO path to DVO prog hardcoded temporarily
@@ -620,8 +654,8 @@
cmd += " -D CATDIR " + self.skychunk.dvoLocation
cmd += " -region "
- cmd += " " + str(raCenter-halfSize)
- cmd += " " + str(raCenter+halfSize)
- cmd += " " + str(decCenter-halfSize)
- cmd += " " + str(decCenter+halfSize)
+ cmd += " " + str(minRA)
+ cmd += " " + str(maxRA)
+ cmd += " " + str(minDEC)
+ cmd += " " + str(maxDEC)
if self.skychunk.parallel:
@@ -632,5 +666,7 @@
p.wait()
- self.ippToPspsDb.setIngestedBox(boxId)
+ ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?
+
+ self.ippToPspsDb.setIngestedBox(boxId, self.scratchDb.dbHost)
# add fileID column
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/dvodetections.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/dvodetections.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/dvodetections.py (revision 35235)
@@ -20,7 +20,7 @@
Constructor
'''
- def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName):
+ def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
- super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
+ super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
# declare DVO file types of interest
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/dvoobjects.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/dvoobjects.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/dvoobjects.py (revision 35235)
@@ -20,7 +20,7 @@
Constructor
'''
- def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDbName=None):
+ def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
- super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDbName)
+ super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
# declare DVO file types of interest
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/dxlayer.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/dxlayer.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/dxlayer.py (revision 35235)
@@ -30,5 +30,5 @@
tempFile = tempfile.NamedTemporaryFile(mode='w+b')
- p = Popen("../perl/deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name,
+ p = Popen("deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name,
shell=True,
stdout=PIPE)
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/gpc1db.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/gpc1db.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/gpc1db.py (revision 35235)
@@ -61,7 +61,9 @@
sql = "SELECT DISTINCT stage_id, DEGREES(ra), DEGREES(decl) \
FROM addRun \
+ JOIN addProcessedExp USING (add_id) \
JOIN minidvodbRun USING(minidvodb_name) \
JOIN minidvodbProcessed USING(minidvodb_id) \
JOIN camRun ON (stage_id = cam_id) \
+ JOIN camProcessedExp USING(cam_id) \
JOIN chipRun USING (chip_id) \
JOIN rawExp USING (exp_id) \
@@ -73,4 +75,6 @@
AND minidvodbProcessed.fault = 0 \
AND mergedvodbProcessed.fault = 0 \
+ AND camProcessedExp.fault = 0 \
+ AND addProcessedExp.fault = 0 \
AND addRun.stage = '" + stage + "' \
AND addRun.state = 'full' \
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopsps.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopsps.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopsps.py (revision 35235)
@@ -40,4 +40,6 @@
self.MINUTES = None
self.SECONDS = None
+
+ self.config.setScratchDbLeaf(self.HOST)
self.logger = self.config.getLogger(self.HOST, self.PID, logToStdout, logToFile)
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopspsdb.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/ipptopspsdb.py (revision 35235)
@@ -4,4 +4,5 @@
import sys
import os
+import math
import logging
@@ -997,6 +998,6 @@
self.skychunk.boxSize = rs.getDouble(10)
- self.skychunk.halfBox = self.skychunk.boxSize/2.0
- self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
+ # self.skychunk.halfBox = self.skychunk.boxSize/2.0
+ # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
self.skychunk.basePath = rs.getString(11)
@@ -1031,13 +1032,24 @@
'''
Gets id of box with these coords
+ XXX EAM : this code implies we can have multiple boxes at the same location for the same
+ XXX EAM : skychunk distinguished by having different sizes. that seems quite bad
'''
def getBoxId(self, ra, dec):
+
+ # for the float comparison, need to use a small window
+ minRA = ra - 0.01
+ maxRA = ra + 0.01
+ minDEC = dec - 0.01
+ maxDEC = dec + 0.01
sql = "SELECT id FROM box \
WHERE skychunk = '" + self.skychunk.name + "' \
- AND ra_center = " + str(ra) + " \
- AND dec_center = " + str(dec) + " \
- AND box_side = " + str(self.skychunk.boxSize)
-
+ AND ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+ AND dec_center BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
+
+ # AND box_side = " + str(self.skychunk.boxSize)
+
+ # print "sql for boxid: ", sql
+
id = -1
try:
@@ -1079,5 +1091,5 @@
Inserts new box into box table. If it already exisits, it returns existing box id
'''
- def isBoxIngested(self, id):
+ def isBoxIngested(self, id, host):
# set the field 'ingested' to 1 for this box
@@ -1088,40 +1100,53 @@
print "no matching boxes for ", str(id)
raise
- status = rs.getInt(1)
+ value = rs.getString(1)
+ except:
+ print "failed to find box for ", str(id)
+ raise
+
+ if value == host:
+ return 1
+
+ if value == "none":
+ return 0
+
+ if value is None:
+ return 0
+
+ print "box ingested host ", value, " does not match db host ", host
+ raise
+
+ '''
+ Inserts new box into box table. If it already exisits, it returns existing box id
+ '''
+ def setIngestedBox(self, id, host):
+
+ # set the field 'ingested' to 1 for this box
+ sql = "update box set ingested = '" + host + "' where id = " + str(id)
+ try:
+ self.execute(sql)
except:
print "failed to set box ", str(id), "as ingested"
raise
- return status
- '''
- Inserts new box into box table. If it already exisits, it returns existing box id
- '''
- def setIngestedBox(self, id):
+ '''
+ Clear ingested flag for boxes owned by this host
+ '''
+ def clearIngestedBoxes(self, host):
# set the field 'ingested' to 1 for this box
- sql = "update box set ingested = 1 where id = " + str(id)
+ sql = "update box set ingested = 'none' where ingested = '" + host + "'"
try:
self.execute(sql)
except:
- print "failed to set box ", str(id), "as ingested"
+ print "failed to clear ingested state for boxes owned by host ", host
raise
'''
- Inserts new box into box table. If it already exisits, it returns existing box id
- '''
- def clearIngestedBoxes(self):
-
- # set the field 'ingested' to 1 for this box
- sql = "update box set ingested = 0"
- try:
- self.execute(sql)
- except:
- print "failed to clear boxes as ingested"
- raise
-
- '''
Inserts new box ids into the pending table
'''
def insertPending(self, box_id, batchType, ids):
+
+ # print "starting insert pending"
# first delete old pending items
@@ -1129,4 +1154,6 @@
WHERE box_id = " + str(box_id) + " \
AND batch_type = '" + batchType + "'")
+
+ # print "deleted old items"
for id in ids:
@@ -1136,4 +1163,5 @@
(" + str(box_id) + ", '" + batchType + "', " + str(id) + ")"
+ # print "sql: ", sql
self.execute(sql)
@@ -1168,5 +1196,5 @@
rs.close()
except:
- self.logger.errorPair("No free stripes", "trying to share with other client...")
+ self.logger.errorPair("No free stripes", "trying to steal boxes from other client...")
sql = "SELECT ra_center, COUNT(*) AS numPending\
FROM box \
@@ -1184,5 +1212,9 @@
rs.close()
except:
- self.logger.errorPair("No available stripes to share", "no stripe obtained")
+ self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
+
+ # tests against ra_center need to use a finite box
+ minRA = raCenter - 0.01
+ maxRA = raCenter + 0.01
self.logger.infoPair("Using RA stripe", "%f" % raCenter)
@@ -1190,5 +1222,5 @@
FROM box \
JOIN pending ON (id = box_id) \
- WHERE ra_center = " + str(raCenter) + " \
+ WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
AND batch_type = '" + batchType + "' \
AND skychunk = '" + self.skychunk.name + "' \
@@ -1203,4 +1235,71 @@
self.registerStripe(host, pid, raCenter)
+
+ self.unlockTables()
+
+ return ids
+
+ '''
+ Returns ids for all boxes with pending items in a stripe of RA not already being processed
+ by another client. If none are available, then it uses the stripe already used by another
+ client that has the most pending items
+ '''
+ def getStripeBoxIdsAnyBatch(self, host, pid):
+
+ self.execute("LOCK TABLES box READ, pending READ, stripe WRITE, clients WRITE")
+
+ ids = []
+ raCenter = -1.0
+ sql = "SELECT ra_center \
+ FROM box \
+ JOIN pending ON (id = box_id) \
+ WHERE skychunk = '" + self.skychunk.name + "' \
+ AND ra_center NOT IN \
+ (SELECT ra_center \
+ FROM stripe \
+ WHERE skychunk = '" + self.skychunk.name + "') \
+ GROUP BY ra_center LIMIT 1"
+
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ raCenter = rs.getFloat(1)
+ rs.close()
+ except:
+ self.logger.errorPair("No free stripes", "trying to steal boxes from other client...")
+ sql = "SELECT ra_center, COUNT(*) AS numPending\
+ FROM box \
+ JOIN pending ON (id = box_id) \
+ WHERE skychunk = '" + self.skychunk.name + "' \
+ GROUP BY ra_center \
+ ORDER BY numPending \
+ DESC LIMIT 1"
+
+ try:
+ rs = self.executeQuery(sql)
+ rs.first()
+ raCenter = rs.getFloat(1)
+ rs.close()
+ except:
+ self.logger.errorPair("No outstanding boxes to steal", "no stripe obtained")
+
+ # tests against ra_center need to use a finite box
+ minRA = raCenter - 0.01
+ maxRA = raCenter + 0.01
+
+ self.logger.infoPair("Using RA stripe", "%f" % raCenter)
+ sql = "SELECT DISTINCT id \
+ FROM box \
+ JOIN pending ON (id = box_id) \
+ WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+ AND skychunk = '" + self.skychunk.name + "' \
+ ORDER BY dec_center DESC"
+
+ try:
+ rs = self.executeQuery(sql)
+ while (rs.next()): ids.append(rs.getInt(1))
+ rs.close()
+ except:
+ self.logger.errorPair("Can't get stripe boxes", sql)
self.unlockTables()
@@ -1250,4 +1349,18 @@
boxDim['DEC'] = rs.getFloat(2)
boxDim['SIDE'] = rs.getFloat(3)
+
+ # SIDE is the linear full-width of the box
+
+ # XXX EAM : need to do something special at the poles...
+ if (boxDim['DEC'] > 85): raise
+
+ # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
+ dR = 0.5*boxDim['SIDE'] / math.cos(math.radians(boxDim['DEC']))
+ dD = 0.5*boxDim['SIDE']
+
+ boxDim['minRA'] = boxDim['RA'] - dR
+ boxDim['maxRA'] = boxDim['RA'] + dR
+ boxDim['minDEC'] = boxDim['DEC'] - dD
+ boxDim['maxDEC'] = boxDim['DEC'] + dD
return boxDim
@@ -1363,11 +1476,12 @@
Gets all items in the all_pending temporary table within the bounds of this box
'''
- def getItemsInThisThisBox(self, ra, dec):
-
- halfSide = self.skychunk.boxSize/2.0
- minRa = ra-halfSide
- maxRa = ra+halfSide
- minDec = dec-halfSide
- maxDec = dec+halfSide
+ def getItemsInThisBox(self, minRA, maxRA, minDEC, maxDEC):
+
+ # the old linear size is ill-conceived
+ ## XX halfSide = self.skychunk.boxSize/2.0
+ ## XX minRa = ra-halfSide
+ ## XX maxRa = ra+halfSide
+ ## XX minDec = dec-halfSide
+ ## XX maxDec = dec+halfSide
ids = []
@@ -1375,6 +1489,6 @@
sql = "SELECT DISTINCT stage_id \
FROM all_pending \
- WHERE ra_bore BETWEEN " + str(minRa) + " AND " + str(maxRa) + " \
- AND dec_bore BETWEEN " + str(minDec) + " AND " + str(maxDec)
+ WHERE ra_bore BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
+ AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
try:
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/loader.py (revision 35235)
@@ -8,4 +8,5 @@
import sys
import os
+import math
import traceback
import logging.config
@@ -43,14 +44,8 @@
# connect to scratch database
- scratchDbs = ['1', '2', '3']
- for dbVersion in scratchDbs:
- self.scratchDb = ScratchDb(self.logger, self.config, dbVersion)
- if not self.config.test and self.scratchDb.anyOtherConnections():
- self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
- self.scratchDb.disconnect()
- continue
- break
-
- if not self.scratchDb.connected:
+ self.scratchDb = ScratchDb(self.logger, self.config)
+ if not self.config.test and self.scratchDb.anyOtherConnections():
+ self.logger.errorPair("This scratch Db is already in use", self.scratchDb.dbName)
+ self.scratchDb.disconnect()
self.exitProgram("Cannot connect to a scratch database")
raise
@@ -97,5 +92,5 @@
except: return ret
- self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
+ self.dvoDetections = DvoDetections(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
return ret
@@ -123,19 +118,27 @@
numAttempts = 0
+ abort = False
while True:
- abort = False
- for batchType in self.skychunk.batchTypes:
-
- # get a stripe's worth of box IDs
- boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType)
-
- numAttempts += 1
-
- self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
- self.logger.infoPair("numAttempts", str(numAttempts))
- if len(boxIds) > 0: numAttempts = 0
-
- for boxId in boxIds:
+ # get a stripe's worth of box IDs
+ boxIds = self.ippToPspsDb.getStripeBoxIdsAnyBatch(self.HOST, self.PID)
+ self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds))
+ self.logger.infoPair("numAttempts", str(numAttempts))
+
+ if len(boxIds) > 0: numAttempts = 0
+ numAttempts += 1
+
+ for boxId in boxIds:
+
+ # get box info. if boxes have changed, break and start again
+ try:
+ boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
+ except:
+ self.logger.infoPair("Can't get details for this box", "%d" % boxId)
+ break
+ self.logger.infoPair("client status", "ok")
+
+ for batchType in self.skychunk.batchTypes:
+
self.logger.infoPair("Check","client status")
if not self.checkClientStatus():
@@ -143,13 +146,7 @@
break
self.logger.infoPair("client status", "ok")
- # get box info. if boxes have changed, break and start again
- try:
- boxDim = self.ippToPspsDb.getBoxDimensions(boxId)
- except:
- self.logger.infoPair("Can't get details for this box", "%d" % boxId)
- break
- self.logger.infoPair("client status", "ok")
+
ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType)
-
+
# ids are the stage_ids for items to be processed (eg, stack_id for a stack CMF, cam_id for P2 smf)
@@ -160,9 +157,7 @@
self.logger.infoSeparator()
self.logger.infoTitle("New box")
- self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE']))
+ self.logger.infoPair("Box dimensions: ", "ra / dec / size : %.1f / %.1f / %.1f -- sky bounds : (%.2f - %2.f, %2.f - %2.f)" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'], boxDim['minRA'], boxDim['maxRA'], boxDim['minDEC'], boxDim['maxDEC']))
self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids))
- boxSizeWithBorder = boxDim['SIDE'] + (self.skychunk.BORDER * 2)
- boxSizeSansBorder = boxDim['SIDE']
- self.logger.infoPair("got here", "ok")
+
useFullTables = 0
if batchType != "OB":
@@ -171,5 +166,8 @@
# look in DVO for this box (with extra border)
self.logger.infoPair("Querying DVO for this sky area", "")
- self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder)
+
+ # XXXX EAM : This code should not be used anymore
+ boxSize = 0.0
+ self.dvoDetections.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSize)
#self.dvoDetections.setSkyArea()
sizeToBeIngested = self.dvoDetections.getDiskSizeOfRegionsToBeIngested()
@@ -191,12 +189,14 @@
useFullTables = 1
else:
- # XXX EAM : this is not currently an
- # option. either remove the non-native
- # ingest code or make it an option
- # if we are using the native loader, always use it
- # need to work out a good boundary / region strategy in coordination with
- # impact of mysql insertion
- if not self.ippToPspsDb.isBoxIngested(boxId):
- self.dvoDetections.nativeIngestDetections(boxId, boxDim['RA'], boxDim['DEC'], boxSizeSansBorder)
+ # XXX EAM : this is not currently optional
+ # (it is the only path). either remove the
+ # non-native ingest code or make it an
+ # option if we are using the native
+ # loader, always use it need to work out a
+ # good boundary / region strategy in
+ # coordination with impact of mysql
+ # insertion
+ if not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
+ self.dvoDetections.nativeIngestDetections(boxId, boxDim)
useFullTables = 1
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/metrics.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/metrics.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/metrics.py (revision 35235)
@@ -25,5 +25,5 @@
self.czarDb = CzarDb(self.logger, self.config)
- self.plot = Plot(self.logger, self.config, self.ippToPspsDb)
+ self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
'''
@@ -37,8 +37,8 @@
now = datetime.datetime.now()
self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S"))
- self.logger.infoPair("Config", self.config.name)
- self.logger.infoPair("Loading epoch", self.config.epoch)
- self.logger.infoPair("DVO label", self.config.dvoLabel)
- self.config.printBoxCoords()
+ self.logger.infoPair("Config", self.skychunk.name)
+ self.logger.infoPair("Loading epoch", self.skychunk.epoch)
+ self.logger.infoPair("DVO label", self.skychunk.dvoLabel)
+ self.skychunk.printBoxCoords()
self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
@@ -77,10 +77,10 @@
self.checkClientStatus()
-
+ self.logger.infoPair("here","yay")
# create a density plot of pending stuff for czartool
self.plot.createDensityPlot("P2", True)
self.plot.createDensityPlot("ST", True)
self.plot.createDensityPlot("OB", True)
-
+ self.logger.infoPair("here","yay2")
# some clean-up before going to sleep
self.ippToPspsDb.purgeDeadClients()
@@ -132,5 +132,5 @@
prevList = success
- self.czarDb.insertStats(stage, self.config.name, batchType, numPending, numSuccess, numFail)
+ self.czarDb.insertStats(stage, self.skychunk.name, batchType, numPending, numSuccess, numFail)
if stage == 'processed': numPendingProcessed = numPending
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/mysql.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/mysql.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/mysql.py (revision 35235)
@@ -84,9 +84,14 @@
connectionID = self.getLastConnectionID()
if connectionID == self.connectionID:
+ print "not killing connection: ", connectionID
self.logger.debug("NOT going to kill THIS connection ID")
return
sql = "KILL %d" % connectionID
- self.execute(sql)
+
+ try:
+ self.execute(sql)
+ except:
+ print "failed to kill connection: ", connectionID
'''
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/objectbatch.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/objectbatch.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/objectbatch.py (revision 35235)
@@ -54,5 +54,6 @@
try:
- self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb.dbName)
+ ## XX pass in the existing connection to scratchDb
+ self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
except:
self.logger.errorPair("Unable to create instance of", "DvoObjects")
@@ -77,4 +78,8 @@
else:
self.dvoObjects.ingestRegion(self.region)
+
+ cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt")
+ if not self.scratchDb.tableExists(cptTableName):
+ return False
return True
@@ -252,4 +257,5 @@
,dataRelease \
,Random \
+ ,batchID \
) \
SELECT \
@@ -268,4 +274,5 @@
, " + str(self.skychunk.dataRelease) + "\
, RAND() \
+ , " + str(self.batchID) + "\
FROM " + cptTableName
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/odm.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/odm.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/odm.py (revision 35235)
@@ -32,5 +32,5 @@
tempFile = tempfile.NamedTemporaryFile(mode='w+b')
- p = Popen("../perl/checkOdmStatus.pl -b " + batch + " -f " + tempFile.name,
+ p = Popen("checkOdmStatus.pl -b " + batch + " -f " + tempFile.name,
shell=True,
stdout=PIPE,
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/plot.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/plot.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/plot.py (revision 35235)
@@ -15,8 +15,9 @@
Constructor
'''
- def __init__(self, logger, skychunk, ippToPspsDb):
+ def __init__(self, logger, skychunk, config, ippToPspsDb):
self.logger = logger
self.skychunk = skychunk
+ self.config = config
self.ippToPspsDb = ippToPspsDb
@@ -25,27 +26,26 @@
'''
def createDensityPlot(self, batchType, forCzartool=False):
-
+
tempFilename = "./" + self.skychunk.name + "_" + batchType + "_plotData.dat"
DATFILE = open(tempFilename,'w')
max = self.ippToPspsDb.createPendingDensityPlotData(batchType, DATFILE)
DATFILE.close()
-
if max == 0: max = 1
timestamp = time.strftime('%Y_%m%d_%H%M%S')
-
+
if forCzartool:
## XXX from config??
- OUTPUTFILE = self.skychunk.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
+ OUTPUTFILE = self.config.czarPlotsPath + "/ippToPsps_density_" + self.skychunk.name + "_" + batchType + ".png"
else:
OUTPUTFILE = self.skychunk.name + "_" + batchType + "_" + timestamp + ".png"
-
+
f = os.popen('/home/panstarrs/ipp/local/bin/gnuplot', 'w')
-
+
if 0:
TERM = "X11"
else:
TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"
-
+ print "output to : "+ OUTPUTFILE
print >> f, "set term " + TERM + "; \
set output \"" + OUTPUTFILE + "\"; \
@@ -57,5 +57,5 @@
set palette rgb 22,13,10; \
set ylabel \"Dec\"; \
- set ylabel \"RA\"; \
+ set xlabel \"RA\"; \
set cbrange [0:" + str(max) + "]; \
set datafile missing \"NaN\"; \
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/plotter.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/plotter.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/plotter.py (revision 35235)
@@ -18,5 +18,5 @@
super(Plotter, self).__init__(argv)
- self.plot = Plot(self.logger, self.skychunk, self.ippToPspsDb)
+ self.plot = Plot(self.logger, self.skychunk, self.config, self.ippToPspsDb)
if len(argv) < 3:
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/queue.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/queue.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/queue.py (revision 35235)
@@ -8,4 +8,5 @@
import sys
import os
+import math
import logging.config
@@ -15,4 +16,5 @@
from dvoobjects import DvoObjects
from batch import Batch
+from scratchdb import ScratchDb
'''
@@ -27,13 +29,26 @@
super(Queue, self).__init__(argv)
+ print "inited ipptopsps"
+
# create various objects
self.gpc1Db = Gpc1Db(self.logger, self.config)
+ print "connect to gpc1"
+
self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
+ print "made datastore"
+
+ # connect to scratch database
+ self.scratchDb = ScratchDb(self.logger, self.config)
+
+ print "connect to scratchdb"
+
try:
- self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb)
+ self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb)
except:
self.exitProgram("Unable to create instance of DvoObject")
raise
+
+ print "got objects"
self.skychunk.printAll()
@@ -56,6 +71,6 @@
for batchType in self.skychunk.batchTypes:
- self.logger.infoTitle("Previous failed datastore loads")
- self.publishAnyUnpublishedBatches(batchType)
+ # self.logger.infoTitle("Previous failed datastore loads")
+ # self.publishAnyUnpublishedBatches(batchType)
# get totals for whole area to check if there is anything to do
@@ -116,13 +131,31 @@
# starting positions
- ra = self.skychunk.minRa + self.skychunk.halfBox
- dec = self.skychunk.minDec + self.skychunk.halfBox
-
- while ra <= self.skychunk.maxRa:
- while dec <= self.skychunk.maxDec:
+
+ dD = 0.5*self.skychunk.boxSize
+ dec = self.skychunk.minDec + dD
+ if (dec > 85): raise
+
+ # print "dec, dD: ", dec, dD
+
+ # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
+ dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ ra = self.skychunk.minRa + dR
+
+ # print "ra, dR: ", ra, dR
+
+ # XXX not sure why this is not done with a for-loop...
+ while dec <= self.skychunk.maxDec:
+ while ra <= self.skychunk.maxRa:
+ # a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
+ minRA = ra - dR
+ maxRA = ra + dR
+ minDEC = dec - dD
+ maxDEC = dec + dD
+
+ # this function inserts the box with given center and (linear) skychunk.boxSize
box_id = self.ippToPspsDb.insertBox(ra, dec)
- ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec)
+ ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC)
self.logger.info("| %5.1f | %5.1f | %9d | %9d |" % (
@@ -132,10 +165,17 @@
len(ids)))
+ # print "box_id: ", box_id
+
if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
-
- dec = dec + self.skychunk.boxSize
- dec = self.skychunk.minDec + self.skychunk.halfBox
- ra = ra + self.skychunk.boxSize
-
+ # print "inserted pending"
+ ra = ra + 2*dR
+ # print "new ra: ", ra
+
+ dec = dec + 2*dD
+ if (dec > 85): raise
+
+ dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ ra = self.skychunk.minRa + dR
+
self.logger.info("+-------------+-------------+-------------+-------------+")
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/scratchdb.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/scratchdb.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/scratchdb.py (revision 35235)
@@ -19,7 +19,9 @@
Constructor
'''
- def __init__(self, logger, config, dbVersion, dbName=None):
+ def __init__(self, logger, config):
# create gpc1 database objects
+ # XXX the test option is probably not needed since
+ # the scratchDb config infor is set up specially
if (config.test):
dbType = "localdatabase_test"
@@ -27,7 +29,7 @@
dbType = "localdatabase"
- if not dbName:
- dbName = config.getDbName(dbType)
- dbName += dbVersion
+ print "trying to connect: ", dbType
+
+ dbName = config.getDbName(dbType)
super(ScratchDb, self).__init__(logger, config, dbType, dbName)
@@ -432,5 +434,6 @@
def purgeTheseDvoRegions(self, regions, fileTypes):
- detectionCount = self.getRowCount(self.dvoDetectionTable)
+ # XXX this must have been a remnant from something else: we are deleting a region, not entries from the dvoDetectionTable
+ # detectionCount = self.getRowCount(self.dvoDetectionTable)
for region in regions:
@@ -448,12 +451,12 @@
pass
- deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable)
+ # deletedDetectionCount = detectionCount - self.getRowCount(self.dvoDetectionTable)
# after a big detection delete, we should OPTIMIZE the table
- if deletedDetectionCount > 0:
- self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable)
- self.optimizeTable(self.dvoDetectionTable)
-
- return deletedDetectionCount
+ # if deletedDetectionCount > 0:
+ # self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable)
+ # self.optimizeTable(self.dvoDetectionTable)
+
+ return 1
'''
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/setupScratchDb.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/setupScratchDb.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/setupScratchDb.py (revision 35235)
@@ -57,5 +57,5 @@
self.config = Config()
- if len(argv) < 2:
+ if len(argv) != 1:
self.printUsage()
self.exitProgram("incorrect args")
@@ -65,4 +65,6 @@
self.PID = os.getpid()
+ self.config.setScratchDbLeaf(self.HOST)
+
# XXX Shouldn't config set up logger?
# (not yet, since we set HOST & PID with program...)
@@ -70,5 +72,5 @@
try:
- self.scratchDb = ScratchDb(self.logger, self.config, argv[1])
+ self.scratchDb = ScratchDb(self.logger, self.config)
except:
self.exitProgram("Could not connect to a scratch Db")
@@ -100,5 +102,5 @@
def printUsage(self):
# write message to log or stdout if no logger set up
- msg = os.path.basename(sys.argv[0]) + " "
+ msg = os.path.basename(sys.argv[0]) + " [-test]"
try:
self.logger.errorPair("Usage:", msg)
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/skychunk.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/skychunk.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/skychunk.py (revision 35235)
@@ -38,5 +38,5 @@
# XXX this is a poor place to put this information -- it is completely gpc-specific.
- # probably should go elsewhere
+ # probably should go elsewhere. This number is scaled in loader.py by cos(dec)
self.BORDER = 1.65
self.isLoaded = False
Index: branches/eam_branches/ipp-20130207/ippToPsps/jython/stackbatch.py
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/jython/stackbatch.py (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/jython/stackbatch.py (revision 35235)
@@ -420,5 +420,5 @@
)"
self.scratchDb.execute(sql)
-
+ self.scratchDb.updateAllRows("StackMeta", "batchID", str(self.batchID))
self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
self.scratchDb.updateFilterID("StackMeta", self.filter)
@@ -846,4 +846,5 @@
#self.scratchDb.makeColumnUnique("StackDetection", "objID")
self.scratchDb.createIndex("StackDetection", "ippDetectID")
+ self.scratchDb.createIndex("StackDetection", "stackDetectID")
self.scratchDb.createIndex("StackApFlx", "ippDetectID")
self.scratchDb.createIndex("StackModelFit", "ippDetectID")
Index: branches/eam_branches/ipp-20130207/ippToPsps/perl/Makefile.am
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/perl/Makefile.am (revision 35235)
+++ branches/eam_branches/ipp-20130207/ippToPsps/perl/Makefile.am (revision 35235)
@@ -0,0 +1,3 @@
+bin_SCRIPTS = checkOdmStatus.pl convertPhotCodesToXml.pl deleteFromDXLayer.pl pspsSchema2xml.pl
+
+INSTALL = @INSTALL@ -C
Index: branches/eam_branches/ipp-20130207/ippToPsps/test/fulltest.sh
===================================================================
--- branches/eam_branches/ipp-20130207/ippToPsps/test/fulltest.sh (revision 35125)
+++ branches/eam_branches/ipp-20130207/ippToPsps/test/fulltest.sh (revision 35235)
@@ -25,7 +25,6 @@
# mkgpc1.sh user localhost eam eam
-set mkgpc1 = 0
+set mkgpc1 = 1
set initdb = 1
-set mkdummy = 1
set initscratch = 1
set camqueue = 1
@@ -34,6 +33,6 @@
set stackqueue = 1
set stackbatch = 1
-set objectqueue = 0
-set objectbatch = 0
+set objectqueue = 1
+set objectbatch = 1
set OUTDIR = "testdata"
@@ -54,52 +53,11 @@
endif
-## XXX no longer needed : setupScratchDb does not need a fake 'config' (or 'chunk')
-## # create a dummy config so setupScratchDb.py does not fail
-## if ($mkdummy) then
-## ippjython queue.py -test edit < 0) then
+ set stackqueuename = $1
+ echo "stackqueue: $stackqueuename"
+endif
+
+set objectqueuename = 0
+if ($#argv > 0) then
+ set stackqueuename = $1
+ echo "stackqueue: $stackqueuename"
+endif
+
+# a bit crude : get the datadir path by interpollation of the PATH element for psconfig
+foreach f ( `echo $PATH | tr : ' '` )
+ echo $f | grep $PSCONFDIR > /dev/null
+ if ($status) continue
+ set datadir = `echo $f | sed s/bin\$/share/`
+ break
+end
+setenv IPPTOPSPS_DATA $datadir/ipptopsps
+
+# before running this test, you must have a mysql user:
+# mkgpc1.sh user localhost eam eam
+
+set mkgpc1 = 0
+set initdb = 0
+set initscratch = 0
+set camqueue = 1
+set initbatch = 0
+set cambatch = 0
+set stackqueue = 0
+set stackbatch = 0
+set objectqueue = 0
+set objectbatch = 0
+
+set OUTDIR = "testdata"
+set queuename = "test2"
+
+if ($mkgpc1) then
+ mkgpc1data.dvo
+endif
+
+# create the ipptopsps database
+if ($initdb) then
+
+ # does the database already exist?
+ # XXX this command will blow away an existing database!!
+ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test"
+ mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql
+
+endif
+
+# re-create the ipptopsps database
+if ($initscratch) then
+
+ # create the database (ignore failure if it already exists)
+ mysql -h localhost -u dvo -pdvo -e "drop database ipptopsps_test_scratch"
+ mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch"
+ ippjython setupScratchDb.py -test
+endif
+
+# this code is sensitive to a previously existing database?
+if ($camqueue) then
+ ippjython queue.py -test edit <