Index: /trunk/ippToPsps/config/tables.DF.vot
===================================================================
--- /trunk/ippToPsps/config/tables.DF.vot (revision 37749)
+++ /trunk/ippToPsps/config/tables.DF.vot (revision 37750)
@@ -283,4 +283,7 @@
flag indicating provenance information
+
+ flag indicating provenance information
+
Data release when this detection was originally taken. Recalibrations do not affect this value.
Index: /trunk/ippToPsps/jython/diffbatch.py
===================================================================
--- /trunk/ippToPsps/jython/diffbatch.py (revision 37749)
+++ /trunk/ippToPsps/jython/diffbatch.py (revision 37750)
@@ -6,5 +6,5 @@
import time
import stilts
-
+import re
from java.lang import *
from java.sql import *
@@ -178,16 +178,16 @@
Populates the Diff table for this OTA
'''
- def populateDiffTable(self, ota, results):
+ def populateDiffDetectionTable(self):
pspsTableName = "DiffDetection"
- ippTableName = "diff_psf"
+ ippTableName = "SkyChip_psf"
- results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
+ #results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName)
# drop then re-create table
- self.scratchDb.dropTable(pspsTableName)
- sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
- try: self.scratchDb.execute(sql)
- except: pass
+ #self.scratchDb.dropTable(pspsTableName)
+ #sql = "CREATE TABLE " + pspsTableName + " LIKE DiffDetection"
+ #try: self.scratchDb.execute(sql)
+ #except: pass
# XXX we no longer delete detections with funny mags (eg PSF_INST_MAG < -17.5)
@@ -195,5 +195,5 @@
BEFORE = self.scratchDb.getRowCount(ippTableName)
- results['SATDET'] = 0
+ #results['SATDET'] = 0
extTimeString = str(self.expTime)
@@ -206,9 +206,11 @@
# XXX apFluxF or apFluxNpix + apFluxRadius?
- sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("ippDetectID", "IPP_IDET")
+ sqlLine.group("diffDetMetaID", str(self.diffSkyFileID));
sqlLine.group("randomDiffID", "FLOOR(RAND("+str(self.batchID)+")*9223372036854775807)")
sqlLine.group("filterID", str(self.filterID))
- sqlLine.group("surveyID", str(self.surveyID))
- ##sqlLine.group("obsTime", str(self.obsTime)) I have expTime
+ sqlLine.group("surveyID", str(self.surveyID))
+ sqlLine.group("skycellID", str(self.skycellID))
+## sqlLine.group("obsTime", str(self.obsTime))
sqlLine.group("expTime", str(self.expTime))
sqlLine.group("xPos", "X_PSF")
@@ -228,8 +230,8 @@
sqlLine.group("psfFlux", "PSF_INST_FLUX / " + extTimeString)
sqlLine.group("psfFluxErr", "PSF_INST_FLUX_SIG / " + extTimeString)
- if (self.id >= 982483):
- sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ")
- sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN")
- sqlLine.group("psfCore", "PSF_CORE")
+# if (self.id >= 982483): #we don't have PSF_FWHM_MAJ or MIN or PSF_CORE
+# sqlLine.group("psfMajorFWHM", "PSF_FWHM_MAJ")
+# sqlLine.group("psfMinorFWHM", "PSF_FWHM_MIN")
+# sqlLine.group("psfCore", "PSF_CORE")
sqlLine.group("psfTheta", "PSF_THETA")
sqlLine.group("psfQf", "PSF_QF")
@@ -249,5 +251,5 @@
sqlLine.group("apFlux", "AP_FLUX / " + extTimeString)
sqlLine.group("apFluxErr", "AP_FLUX_SIG / " + extTimeString)
- sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
+ # sqlLine.group("apFillF", "AP_NPIX / (3.14159265359 * POW(AP_MAG_RADIUS - 0.5, 2))")
# sqlLine.group("apRadius", "AP_MAG_RADIUS")
sqlLine.group("kronFlux", "KRON_FLUX / " + extTimeString)
@@ -291,8 +293,22 @@
# we don't delete these anymore
- results['NULLINSTFLUX'] = 0;
+ #results['NULLINSTFLUX'] = 0;
# insert stuff from dvo
+ self.logger.infoPair("inserting dvo info:find imageID"," from externID")
+ imageID = self.scratchDb.getImageIDFromExternID(self.diffSkyFileID)
+ self.logger.infoPair("updating","diffdetection")
+ sqlLine = sqlUtility("UPDATE DiffDetection as a, " + self.scratchDb.dvoDetectionTable + " as b SET ")
+ sqlLine.group("a.diffObjID", "b.objId")
+
+ sql = sqlLine.makeEquals("WHERE a.ippDetectID = b.ippDetectID AND b.imageID = " +str( imageID))
+
+ try: self.scratchDb.execute(sql)
+ except:
+ self.logger.errorPair("failed sql:",sql)
+ raise
+ self.logger.infoPair("need to cull"," nulls objid")
+ self.logger.infoPair("need to add","uniquePSPSDFid")
@@ -304,5 +320,5 @@
self.logger.infoPair("Procesing table", "DiffToImage")
- imageIDs = self.gpc1Db.getImageIDsForThisDiffID(diffkmetaID)
+ imageIDs = self.gpc1Db.getImageIDsForThisDiffID(self.diffSkyFileID)
for imageID in imageIDs:
@@ -310,5 +326,5 @@
sql = "INSERT INTO DiffToImage (diffMetaID, imageID) \
VALUES (\
- " + str(diffmetaID) + ", " + imageID + ")"
+ " + str(self.diffSkyFileID) + ", " + imageID + ")"
self.scratchDb.execute(sql)
@@ -335,5 +351,5 @@
self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
- self.populateDiffMeta()
+ #self.populateDiffMeta()
# dictionary objects to hold imageIDs for later
@@ -355,7 +371,7 @@
self.logger.infoPair("Creating indexes on", "IPP tables")
- self.scratchDb.createIndex(extension, "IPP_IDET")
+ #self.scratchDb.createIndex(extension, "IPP_IDET")
# try the test Chip
- self.scratchDb.createIndex("Chip_psf", "IPP_IDET")
+ self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET")
self.logger.infoPair("created indexes on", "IPP tables")
@@ -391,7 +407,7 @@
def updatePspsUniqueIDs(self,table):
- sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, objID from \
+ sql = "UPDATE "+table+" join (select @r:=@r+1 rownum, diffobjID from \
(select @r:=0) r, "+table+" t) as foo using (objID) set \
- uniquePspsP2id = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
+ uniquePspsDFid = ((" +str(self.batchID)+ "*1000000000 ) + rownum)"
try: self.scratchDb.execute(sql)
except:
@@ -402,85 +418,4 @@
Does the processing, i.e. pulling stuff from IPP tables into PSPS tables
'''
- def populatePspsTablesChip(self, chipname, x, y, results, tables):
- # XXX EAM NOTE: drop tables Detection_* here so
- # they do not polute the db?
- # XXX or put an explicit drop at the end of the loop?
-
- # self.logger.infoTitle("Processing " + chipname)
- # this is a bit crude: if the chip is not present, this test will fail and the chip
- # will be (correctly) skipped. would be better to carry that information explicitly ("chip is missing")
-
- # XXX keep this in or not?
- # if not self.scratchDb.astrometricSolutionOK(chipname):
- # self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % chipname)
- # return False
-
- # does this chip exist in the DVO image table (if NOT, then skip it)
- if not self.scratchDb.haveThisChip(self.imageIDs[chipname], self.sourceIDs[chipname]):
- self.logger.info("| %5s | ------------- Chip not in DVO : rejecting ------------ |" % chipname)
- return False
-
- #self.logger.info("populate stuff ");
- # populate remainder of tables
- self.populateDetectionTable(chipname, results)
- #self.logger.info("successful populate ");
- # now add DVO IDs
- self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname])
- #self.logger.info("updated dvoids")
- results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID")
- #self.logger.info("deleted nulls")
- self.logger.info("add psps unique p2 ids")
- self.updatePspsUniqueIDs("Detection_" + chipname, self.imageIDs[chipname])
- self.updateImageID("Detection_" + chipname, x, y)
- #self.logger.info("updateImageId")
- rowCount = self.scratchDb.getRowCount("Detection_" + chipname)
- #self.logger.info("got row count")
- self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |",
- chipname,
- results['ORIGINALTOTAL'],
- results['SATDET'],
- results['NULLINSTFLUX'],
- results['NULLOBJID'],
- rowCount)
- self.totalOriginal = self.totalOriginal + results['ORIGINALTOTAL']
- self.totalSatDet = self.totalSatDet + results['SATDET']
- self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX']
- self.totalNullObjID = self.totalNullObjID + results['NULLOBJID']
- self.totalDetections = self.totalDetections + rowCount
-
- fractionKept = float(rowCount) / float(results['ORIGINALTOTAL'])
- # print "kept ", str(fractionKept), " detections from ", chipname
-
- if fractionKept < 0.8:
- # print "Too many rejected detections (", str(fractionKept), ", fail on exposure for", chipname
- self.logger.infoPair("Too many rejected detections, fail on exposure for", chipname)
- self.skipBatch = True
- return False;
-
- #self.logger.info("updated totals")
- # check we have something in this Detection table TODO add this to table above
- if rowCount < 1:
- self.logger.infoPair("Empty table for chip, fail on exposure", chipname)
- self.skipBatch = True
- return False;
-
- # update ImageMeta with count of detections for this CHIPNAME and photoCodeID
- sql = "UPDATE ImageMeta_" + chipname + " \
- SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + chipname), self.scratchDb.getPhotoCalID(self.imageIDs[chipname]))
- self.scratchDb.execute(sql)
-
- # add these to list of tables to export later
- self.tablesToExport.append("ImageMeta_" + chipname)
- self.logger.info("export ImageMeta")
-
- self.tablesToExport.append("Detection_" + chipname)
- self.logger.info("export Detection")
-
- tables.append("Detection_" + chipname)
- self.logger.info("updated detection")
-
- self.validChips.append(chipname)
-
- return True
'''
@@ -494,9 +429,11 @@
self.tablesToExport.append("DiffMeta")
self.populateDiffMeta()
+ self.logger.infoPair("populated","DiffMeta")
self.tablesToExport.append("DiffToImage")
self.populateDiffToImage()
+ self.logger.infoPair("populated","DiffToImage")
self.tablesToExport.append("DiffDetection")
- self.populateDiffDetection()
-
+ self.populateDiffDetectionTable()
+ self.logger.infoPair("populated","DiffDetection")
@@ -575,25 +512,61 @@
else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials)
-
- '''
- This function reads the cmf/smf file and loads it into the database as its own table
- Overriding this method. Filter to only import *.psf extensions
- '''
- def importIppTables(self, filter=""):
-
- regex = ".*.psf"
- if False and self.config.test and self.config.camera == "gpc1":
- regex = "XY01.psf"
+ '''
+ Imports IPP tables from FITS file
+ Accepts a regular expression to match the tables so not all tables need to be imported
+ (This regex feature is not currently used...)
+ '''
+ def importIppTables(self, columns="*", tableRE=""):
+
+ if self.config.retry: return True
+
+ self.logger.infoPair("Importing DF tables with table match expression: ", tableRE)
+
+ count = 0
+ fileName = self.fits.getPath()
- print "my ID: " + str(self.id)
-
- if (self.id < 982483):
- # XXX EAM NOTE : this is fragile : requires PS1_V4
- columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_MAJOR PSF_MINOR PSF_THETA PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
- else:
- # XXX EAM NOTE : this is fragile : requires PS1_V5
- columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG POSANGLE PLTSCALE PSF_INST_FLUX PSF_INST_FLUX_SIG PSF_FWHM_MAJ PSF_FWHM_MIN PSF_THETA PSF_CORE PSF_QF PSF_QF_PERFECT PSF_CHISQ EXT_NSIGMA MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_FLUX AP_FLUX_SIG AP_NPIX AP_MAG_RADIUS KRON_FLUX KRON_FLUX_ERR SKY SKY_SIGMA FLAGS FLAGS2"
-
- return super(DetectionBatch, self).importIppTables(columns, regex)
+ self.logger.infoPair("using filename:",fileName)
+
+ try:
+ tables = stilts.treads(fileName)
+ except:
+ self.logger.errorPair("STILTS could not import from", fileName)
+ return False
+
+ # count = 0
+ for table in tables:
+
+ match = re.match(tableRE, table.name)
+ if not match: continue
+
+ self.logger.infoPair("Reading IPP table", table.name)
+ table = stilts.tpipe(table, cmd='addcol table_index $0')
+ table = stilts.tpipe(table, cmd='explodeall')
+
+ # drop any previous tables before import
+ self.scratchDb.dropTable(table.name)
+
+
+ # IPP FITS files are littered with infinities, so remove these
+ self.logger.info("Removing Infinity values from all columns")
+ table = stilts.tpipe(table, cmd='keepcols "' + columns + '"')
+ table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
+ table = stilts.tpipe(table, cmd='replaceval Infinity null *')
+
+ try:
+ table.write(self.scratchDb.url + '#' + table.name)
+ count = count + 1
+ except:
+ self.logger.exception("Problem writing table '" + table.name + "' to the database")
+
+ self.logger.infoPair("Done. Imported", "%d tables" % count)
+ self.indexIppTables()
+
+ return True
+
+
+
+
+
'''
@@ -601,3 +574,12 @@
'''
def exportPspsTablesToFits(self, regex="(.*)"):
- return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+ return super(DiffBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+
+
+ '''
+ Drop a table and report hte drop
+ '''
+ def dropTableVerbose(self, table):
+ self.logger.infoPair("dropping table:",table)
+ self.scratchDb.dropTable(table)
+
Index: /trunk/ippToPsps/jython/dvodiffdetections.py
===================================================================
--- /trunk/ippToPsps/jython/dvodiffdetections.py (revision 37750)
+++ /trunk/ippToPsps/jython/dvodiffdetections.py (revision 37750)
@@ -0,0 +1,167 @@
+#!/usr/bin/env jython
+
+import stilts
+import datetime
+import re
+import sys
+import os
+import logging
+import glob
+
+from dvo import Dvo
+
+
+'''
+A class for ingesting DVO detections into MySQL
+'''
+class DvoDetections(Dvo):
+
+ '''
+ Constructor
+ '''
+ def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
+
+ super(DvoDetections, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
+
+ # declare DVO file types of interest
+ self.ingestFileTypes = ['cpm','cpt']
+
+ '''
+ Ingests detections for this region
+ '''
+ def ingestRegion(self, region):
+
+ cpmPath = self.skychunk.dvoLocation + "/" + region + ".cpm"
+ cptPath = self.skychunk.dvoLocation + "/" + region + ".cpt"
+
+ cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm")
+ cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt")
+
+ # if we have not already ingested the cpm table, then do it now
+ if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath):
+
+ # import cpm table and index
+ self.importFits(
+ cpmPath,
+ "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS PHOTCODE AIRMASS M_TIME M_CAL MAG_CAL_ERR D_RA D_DEC PLTSCALE X_CCD_ERR Y_CCD_ERR",
+ cpmTableName)
+
+ self.scratchDb.createIndex(cpmTableName, "IMAGE_ID")
+ self.scratchDb.createIndex(cpmTableName, "CAT_ID")
+ self.scratchDb.createIndex(cpmTableName, "OBJ_ID")
+
+ # XXX: Should we create index on PHOTCODE?
+
+ # we can now report that we have imported this table
+ self.scratchDb.setImportedThisDvoTable(cpmPath, region)
+
+ # get the id for this cpm file so we can insert it in the dvoDetection table
+ fileId = self.scratchDb.getDvoFileId(cpmPath)
+
+ # import cpt table and index
+ self.importFits(
+ cptPath,
+ "OBJ_ID CAT_ID EXT_ID RA DEC",
+ cptTableName)
+
+ self.scratchDb.createIndex(cptTableName, "IMAGE_ID")
+ self.scratchDb.createIndex(cptTableName, "CAT_ID")
+ self.scratchDb.createIndex(cptTableName, "OBJ_ID")
+
+ # SA10
+ # Columns needed for DetectionCalib and StackDetectionCalib
+ # zp = photcode.K * (AIRMASS - 1) + photcode.C_LAM * 0.001 - cpm.M_CAL
+ # zpErr = cpm.MAG_CAL_ERR
+ # airMass = cpm.AIRMASS
+ # expTime = POW(10.0, 0.4 * cpm.M_TIME)
+ # ra = cpt.RA - (cpm.D_RA / 3600.0)
+ # dec = cpt.DEC - (cpm.D_DEC / 3600.0)
+ # raErr = cpm.X_CCD_ERR * cpm.PLTSCALE
+ # decErr = cpm.Y_CCD_ERR * cpm.PLTSCALE
+
+ self.logger.infoPair("Adding","PSPS objIDs")
+ # first try to add the new columns. catch failure and continue
+ try:
+ sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT), ADD COLUMN (ZP REAL), ADD COLUMN (RA FLOAT), ADD COLUMN (DEC_ FLOAT)"
+ self.scratchDb.execute(sql)
+ except:
+ # failed, but may be due to mysql error
+ self.logger.infoPair("mysql error or already added PSPS_OBJ_ID column to", cpmTableName)
+ # self.logger.infoPair("sql command", sql)
+
+ # shove PSPS objIDs from cpt table and the calibrated zero point and coordinates which are formed
+ # by combining values from multiple tables into cpt table
+ sql = "UPDATE " \
+ + cpmTableName + " AS meas, " \
+ + cptTableName + " AS ave, " \
+ + self.scratchDb.dvoPhotcodesTable + " AS phot \
+ SET \
+ meas.PSPS_OBJ_ID = ave.EXT_ID, \
+ meas.ZP = phot.C_LAM * 0.001 + phot.K * (meas.AIRMASS - 1) - meas.M_CAL, \
+ meas.RA = ave.RA - (meas.D_RA / 3600.0), \
+ meas.DEC_ = ave.DEC_ - (meas.D_DEC / 3600.0) \
+ WHERE meas.CAT_ID = ave.CAT_ID \
+ AND meas.OBJ_ID = ave.OBJ_ID \
+ AND meas.PHOTCODE = phot.CODE"
+
+ # EAM : add some debug I/O
+ # self.logger.infoPair("sql command", sql)
+ self.scratchDb.execute(sql)
+
+ # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm
+ # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different
+ # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary
+ # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest
+ # XXX: Bills 2012-10031 Does this problem still exist?
+ self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable)
+ sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\
+ imageID \
+ ,ippDetectID \
+ ,detectID \
+ ,ippObjID \
+ ,objID \
+ ,flags \
+ ,zp \
+ ,zpErr \
+ ,airMass \
+ ,expTime \
+ ,ra \
+ ,dec_ \
+ ,raErr \
+ ,decErr \
+ ,fileID \
+ ) SELECT \
+ IMAGE_ID \
+ ,DET_ID \
+ ,EXT_ID \
+ ,CAT_ID * 1000000000 + OBJ_ID \
+ ,PSPS_OBJ_ID \
+ ,DB_FLAGS \
+ ,ZP \
+ ,MAG_CAL_ERR \
+ ,AIRMASS \
+ ,POW(10.0, 0.4 * M_TIME) \
+ ,RA \
+ ,DEC_ \
+ ,X_CCD_ERR*PLTSCALE \
+ ,Y_CCD_ERR*PLTSCALE \
+ , " + str(fileId) + " \
+ FROM " + cpmTableName
+
+ # print "sql :", sql
+ # response = raw_input("prepare to insert dvoDetectionTable ")
+
+ try:
+ self.scratchDb.execute(sql)
+ except:
+ self.logger.error("FAILED: " + sql)
+ return False
+ finally:
+ # response = raw_input("insert dvoDetectionTable ")
+ self.logger.infoPair("Dropping tables", "%s and %s" % (cpmTableName, cptTableName))
+ self.scratchDb.dropTable(cpmTableName)
+ self.scratchDb.dropTable(cptTableName)
+
+ self.scratchDb.setImportedThisDvoTable(cptPath, region)
+
+ return True
Index: /trunk/ippToPsps/jython/dvodiffobjects.py
===================================================================
--- /trunk/ippToPsps/jython/dvodiffobjects.py (revision 37750)
+++ /trunk/ippToPsps/jython/dvodiffobjects.py (revision 37750)
@@ -0,0 +1,66 @@
+#!/usr/bin/env jython
+
+import stilts
+import datetime
+import re
+import sys
+import os
+import logging
+import glob
+from subprocess import call, PIPE, Popen
+
+from dvo import Dvo
+
+'''
+A class for ingesting DVO detections into MySQL
+'''
+class DvoObjects(Dvo):
+
+ '''
+ Constructor
+ '''
+ def __init__(self, logger, config, skychunk, ippToPspsDb, scratchDb):
+
+ super(DvoObjects, self).__init__(logger, config, skychunk, ippToPspsDb, scratchDb)
+
+ # declare DVO file types of interest
+ self.ingestFileTypes = ['cpt', 'cps']
+ self.loadPhotcodes()
+
+ '''
+ ingest object data into MySQL database using the native DVO program dvopsps
+ Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this
+ includes purging detections outside sky area
+ '''
+ def nativeIngestRegion(self, region):
+
+ # drop cpt/cps/object table?
+ # create detections table?
+
+ # TODO path to DVO prog hardcoded temporarily
+ cmd = "dvopsps objects"
+ cmd += " -dbhost " + self.scratchDb.dbHost
+ cmd += " -dbname " + self.scratchDb.dbName
+ cmd += " -dbuser " + self.scratchDb.dbUser
+ cmd += " -dbpass " + self.scratchDb.dbPass
+ cmd += " -D CATDIR " + self.skychunk.dvoLocation
+ cmd += " -cpt " + region
+
+ if self.skychunk.parallel:
+ if self.scratchDb.dbHost == "localhost":
+ self.logger.info("dvopsps objects -parallel is incompatible with localhost for scratchdb")
+ raise
+
+ cmd += " -parallel"
+
+ self.logger.infoPair("Running dvopsps", cmd)
+ p = Popen(cmd, shell=True, stdout=PIPE)
+ p.wait()
+ if p.returncode != 0:
+ self.logger.errorPair("dvopsps failed on ", region)
+ raise
+
+ # update lists after attempted sync
+ # self.printSummary()
+
+ return True
Index: /trunk/ippToPsps/jython/gpc1db.py
===================================================================
--- /trunk/ippToPsps/jython/gpc1db.py (revision 37749)
+++ /trunk/ippToPsps/jython/gpc1db.py (revision 37750)
@@ -354,11 +354,16 @@
FROM warpSkyCellMap \
JOIN warpRun USING(warp_id) \
- JOIN diffInputSkyfile on warp1 = warp_id \
- WHERE diffInputSkyfile.diff_skyfile_id = " + str(diffID) + " AS a"
-
- try:
- rs = self.executeQuery(sql)
- except:
+ JOIN diffInputSkyfile on warp1 = warp_id \
+ JOIN fakeRun using (fake_id) \
+ JOIN camRun using (cam_id) \
+ JOIN chipRun using (chip_id) \
+ WHERE diffInputSkyfile.diff_skyfile_id = " + str(diffID) + ") AS a"
+
+ try:
+ rs = self.executeQuery(sql)
+ except:
+ self.logger.infoPair("failed sql:",sql)
self.logger.exception("Can't query for imageIDs")
+
imageIDs = []