Index: /branches/eam_branches/ipp-20130509/doc/stamps/stampsICD.tex
===================================================================
--- /branches/eam_branches/ipp-20130509/doc/stamps/stampsICD.tex (revision 35747)
+++ /branches/eam_branches/ipp-20130509/doc/stamps/stampsICD.tex (revision 35748)
@@ -310,7 +310,9 @@
PSTAMP\_SELECT\_MASK & 2 & create postage stamp of the mask pixels \\
PSTAMP\_SELECT\_VARIANCE & 4 & create postage stamp of the variance pixels \\
-PSTAMP\_SELECT\_CMF & 8 & return the sources file (.cmf) for the images' run \\
+PSTAMP\_SELECT\_SOURCES & 8 & extract sources from the region of interest from the sources file (.cmf) for the images' run \\
PSTAMP\_SELECT\_PSF & 16 & return the psf file for the images' run \\
PSTAMP\_SELECT\_BACKMDL & 32 & return the background model file for the images chipRun (applies only for chip) \\
+PSTAMP\_SELECT\_JPEG & 64 & return a jpeg format image of the stamp \\
+PSTAMP\_SELECT\_UNCOMPRESSED & 512 & do not use FITS image compression \\
PSTAMP\_SELECT\_INVERSE & 1024 & make stamps from inverse difference images (applies only for diff) \\
PSTAMP\_SELECT\_UNCONV & 2048 & make stamps from unconvolved stack images (applies only for stack) \\
Index: /branches/eam_branches/ipp-20130509/ippTasks/survey.pro
===================================================================
--- /branches/eam_branches/ipp-20130509/ippTasks/survey.pro (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippTasks/survey.pro (revision 35748)
@@ -44,5 +44,5 @@
$SURVEY_TIMEOUT = 600
# the tasks for loading the release tabels for nightly science do not need to run very often
-$SURVEY_RELEASE_EXEC 300
+$SURVEY_RELEASE_EXEC = 300
macro survey.on
@@ -382,10 +382,11 @@
macro survey.add.chip.bg
if ($0 != 4)
- echo "USAGE: survey.add.chip.bg (label) (dist_group) (muggle)"
+ echo "USAGE: survey.add.chip.bg (label) (cam_label) (dist_group)"
break
end
book newpage SURVEY_CHIP_BG $1
- book setword SURVEY_CHIP_BG $1 DIST_GROUP $2
- book setword SURVEY_CHIP_BG $1 MUGGLE $3
+ book setword SURVEY_CHIP_BG $1 LABEL $1
+ book setword SURVEY_CHIP_BG $1 CAM_LABEL $2
+ book setword SURVEY_CHIP_BG $1 DIST_GROUP $3
book setword SURVEY_CHIP_BG $1 STATE PENDING
end
@@ -410,10 +411,11 @@
macro survey.add.warp.bg
if ($0 != 4)
- echo "USAGE: survey.add.warp.bg (label) (dist_group) (muggle)"
+ echo "USAGE: survey.add.warp.bg (label) (warp_label) (dist_group)"
break
end
book newpage SURVEY_WARP_BG $1
- book setword SURVEY_WARP_BG $1 DIST_GROUP $2
- book setword SURVEY_WARP_BG $1 MUGGLE $3
+ book setword SURVEY_WARP_BG $1 LABEL $1
+ book setword SURVEY_WARP_BG $1 WARP_LABEL $2
+ book setword SURVEY_WARP_BG $1 DIST_GROUP $3
book setword SURVEY_WARP_BG $1 STATE PENDING
end
@@ -1312,11 +1314,12 @@
book setword SURVEY_CHIP_BG $label STATE DONE
+ book getword SURVEY_CHIP_BG $label CAM_LABEL -var cam_label
book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group
- book getword SURVEY_CHIP_BG $label MUGGLE -var muggle
+# book getword SURVEY_CHIP_BG $label MUGGLE -var muggle
- $run = bgtool -definechip -label $label -set_dist_group $dist_group
- if ($muggle == 0)
- $run = $run -destreaked
- end
+ $run = bgtool -definechip -label $label -cam_label $cam_label -set_dist_group $dist_group
+# if ($muggle == 0)
+# $run = $run -destreaked
+# end
if ($DB:n == 0)
@@ -1388,11 +1391,11 @@
book setword SURVEY_WARP_BG $label STATE DONE
+ book getword SURVEY_WARP_BG $label WARP_LABEL -var warp_label
book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group
- book getword SURVEY_WARP_BG $label MUGGLE -var muggle
- $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group
- if ($muggle == 0)
- $run = $run -destreaked
- end
+ $run = bgtool -definewarp -set_label $label -chip_bg_label $label -warp_label $warp_label -set_dist_group $dist_group
+# if ($muggle == 0)
+# $run = $run -destreaked
+# end
Index: /branches/eam_branches/ipp-20130509/ippToPsps/config/settings.xml
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/config/settings.xml (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/config/settings.xml (revision 35748)
@@ -6,6 +6,5 @@
- logs
-
+ SAS15
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/detectionbatch.py
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/detectionbatch.py (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/detectionbatch.py (revision 35748)
@@ -450,5 +450,5 @@
,MOMENTS_XY \
,MOMENTS_YY \
- ,AP_FLUX / " + self.header['EXPTIME'] + " \
+ ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \
,KRON_FLUX / " + self.header['EXPTIME'] + " \
,KRON_FLUX_ERR / " + self.header['EXPTIME'] + " \
@@ -469,5 +469,7 @@
# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + "
# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + self.header['EXPTIME'] + ")) / 1.085736)
-# ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + "
+
+# I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX
+# AP_FLUX is no ,AP_FLUX / " + self.header['EXPTIME'] + "
self.scratchDb.execute(sql)
@@ -1035,5 +1037,8 @@
# XXX EAM NOTE : this is fragile : requires PS1_V4
- columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
+ # columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG AP_FLUX KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
+
+ # XXX EAM NOTE : this is fragile : requires PS1_V4
+ columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ"
return super(DetectionBatch, self).importIppTables(columns, regex)
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/dvo.py
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/dvo.py (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/dvo.py (revision 35748)
@@ -65,12 +65,11 @@
if not self.correctDvo:
print "*******************************************************************************"
- self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.skychunk.dvoLocation + "'?")
- response = raw_input("(y/n) ")
- if response == "y":
- self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
- response = raw_input("(y/n) ")
- if response == "y":
- self.resetAllTables()
-
+ self.logger.warning("switching to a new dvo: " + self.skychunk.dvoLocation)
+ # response = raw_input("(y/n) ")
+ # if response == "y":
+ # self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
+ # response = raw_input("(y/n) ")
+ # if response == "y":
+ self.resetAllTables()
'''
@@ -597,13 +596,14 @@
self.ippToPspsDb.clearIngestedBoxes(self.scratchDb.dbHost)
- ## print "*******************************************************************************"
- ## self.logger.warning("about to delete detection table and re-ingest; is this correct?")
- ## response = raw_input("(y/n) ")
- ## if response != "y":
- ## raise
- ## self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
- ## response = raw_input("(y/n) ")
- ## if response != "y":
- ## raise
+ if True:
+ print "*******************************************************************************"
+ self.logger.warning("about to delete detection table and re-ingest; is this correct?")
+ #response = raw_input("(y/n) ")
+ #if response != "y":
+ # raise
+ #self.logger.warning("Are you ABSOLUTELY sure you want to do this?")
+ #response = raw_input("(y/n) ")
+ #if response != "y":
+ # raise
# blow away existing dvoDetection table & re-crate
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/ippjython
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/ippjython (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/ippjython (revision 35748)
@@ -19,4 +19,5 @@
set JYTHON = $JARDIR/jython2.5.3b1/jython.jar
# set JYTHON = ~/jython2.5.3b1/jython.jar
+#set JYTHON = $JARDIR/jython.jar
if (! -r $JYTHON) then
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/ipptopspsdb.py (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/ipptopspsdb.py (revision 35748)
@@ -785,5 +785,6 @@
survey, \
dvo_db, \
- datastore_product \
+ datastore_product, \
+ comment \
) VALUES ( \
'" + batchType + "', \
@@ -791,5 +792,6 @@
'" + self.skychunk.survey + "', \
'" + self.skychunk.dvoLabel + "', \
- '" + self.skychunk.datastoreProduct + "' \
+ '" + self.skychunk.datastoreProduct + "', \
+ '" + self.skychunk.name + "' \
)"
self.logger.infoPair("heather:","sql")
@@ -1215,4 +1217,6 @@
('" + self.skychunk.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.skychunk.boxSize) + ")"
+ # print "insertBox: ", sql
+
self.execute(sql)
id = self.getBoxId(ra, dec)
@@ -1556,6 +1560,6 @@
sql = "SELECT id, ra_center, dec_center \
- FROM box \
- WHERE skychunk = '" + self.skychunk.name + "' \
+ FROM box join skychunk on box.skychunk = skychunk.name \
+ WHERE dvo_label = '" + self.skychunk.dvoLabel + "' \
ORDER BY ra_center, dec_center"
@@ -1599,4 +1603,5 @@
sql = "INSERT INTO all_pending (stage_id, ra_bore, dec_bore) \
VALUES (%d, %f, %f)" % (row[0], row[1], row[2])
+ # print "sql 1: ", sql
self.execute(sql)
except: continue
@@ -1604,4 +1609,21 @@
count = self.getRowCount("all_pending")
self.logger.infoPair("Items written to Db", "%d" % count)
+
+ # try:
+ # self.execute("DROP TABLE all_pending_alt")
+ # except:
+ # pass
+ #
+ # self.execute("CREATE TABLE all_pending_alt (stage_id bigint(20), ra_bore float, dec_bore float)")
+ # for row in rows:
+ # try:
+ # sql = "INSERT INTO all_pending_alt (stage_id, ra_bore, dec_bore) \
+ # VALUES (%d, %f, %f)" % (row[0], row[1], row[2])
+ # print "sql 2: ", sql
+ # self.execute(sql)
+ # except: continue
+ #
+ # count = self.getRowCount("all_pending_alt")
+ # self.logger.infoPair("Items written to alt Db", "%d" % count)
'''
@@ -1624,4 +1646,6 @@
AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
+ print "items: ", sql
+
try:
rs = self.executeQuery(sql)
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/queue.py
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/queue.py (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/queue.py (revision 35748)
@@ -37,10 +37,10 @@
self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb)
- # print "made datastore"
+ print "made datastore"
# connect to scratch database
self.scratchDb = ScratchDb(self.logger, self.config)
- # print "connect to scratchdb"
+ print "connect to scratchdb"
try:
@@ -50,5 +50,5 @@
raise
- # print "got objects"
+ print "got objects"
self.skychunk.printAll()
@@ -62,13 +62,13 @@
'''
def run(self):
-
+ print "here"
# this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time)
while True:
self.ippToPspsDb.removeAllBoxes()
-
+ print "remove boxes"
# queue up batches that are processed but not loaded to datastore
for batchType in self.skychunk.batchTypes:
-
+ self.logger.infoPair("BatchType",batchType)
# self.logger.infoTitle("Previous failed datastore loads")
# self.publishAnyUnpublishedBatches(batchType)
@@ -76,6 +76,7 @@
# get totals for whole area to check if there is anything to do
processedIDs = self.ippToPspsDb.getProcessedIDs(batchType)
+ print "got processed ids"
consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType)
-
+ print "got here"
# for object batches, get full list of stuff from Dvo
# XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo
@@ -133,19 +134,24 @@
dD = 0.5*self.skychunk.boxSize
+
dec = self.skychunk.minDec + dD
+ #dec = self.skychunk.minDec
+
if (dec > 85): raise
- # print "dec, dD: ", dec, dD
+ print "dec, dD: ", dec, dD ,self.skychunk.maxDec
# a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
- dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ dR = 0.5*self.skychunk.boxSize
+
ra = self.skychunk.minRa + dR
- # print "ra, dR: ", ra, dR
+ print "ra, dR, maxRa", ra, dR, self.skychunk.maxRa
# XXX not sure why this is not done with a for-loop...
while dec <= self.skychunk.maxDec:
while ra <= self.skychunk.maxRa:
-
+ print "here"
# a given box centered at RA,DEC with (linear) size of SIDE has the following bounds
minRA = ra - dR
@@ -165,5 +171,5 @@
len(ids)))
- # print "box_id: ", box_id
+ print "box_id: ", box_id
if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
@@ -175,5 +181,6 @@
if (dec > 85): raise
- dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ #dR = 0.5*self.skychunk.boxSize / math.cos(math.radians(dec))
+ dR = 0.5*self.skychunk.boxSize
ra = self.skychunk.minRa + dR
Index: /branches/eam_branches/ipp-20130509/ippToPsps/jython/stackbatch.py
===================================================================
--- /branches/eam_branches/ipp-20130509/ippToPsps/jython/stackbatch.py (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ippToPsps/jython/stackbatch.py (revision 35748)
@@ -455,4 +455,5 @@
,psfQf \
,psfQfPerfect \
+ ,psfChiSq \
,momentXX \
,momentXY \
@@ -469,5 +470,4 @@
,kronFluxErr \
,kronRad \
- ,psfChiSq \
,nFrames \
,assocDate \
@@ -495,4 +495,5 @@
,PSF_QF \
,PSF_QF_PERFECT \
+ ,PSF_CHISQ / PSF_NDOF \
,MOMENTS_XX \
,MOMENTS_XY \
@@ -504,10 +505,9 @@
,MOMENTS_R1 \
,MOMENTS_RH \
- ,AP_FLUX / " + str(self.expTime) + " \
+ ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \
,NULL \
,KRON_FLUX / " + str(self.expTime) + " \
,KRON_FLUX_ERR / " + str(self.expTime) + " \
,MOMENTS_R1 * 2.5 \
- ,PSF_CHISQ / PSF_NDOF \
,N_FRAMES \
, '" + self.dateStr + "' \
@@ -520,4 +520,7 @@
# ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + "
+# I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX
+# ,AP_FLUX / " + str(self.expTime) + "
+
# print "sql: ", sql
# response = raw_input("ready to insert stack det ")
@@ -540,5 +543,5 @@
self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.skychunk.dataRelease))
-
+ #set primary F to 0
self.scratchDb.updateAllRows("StackDetection", "primaryF", "0")
@@ -550,4 +553,5 @@
self.updateDvoIDsAndFlags("StackDetection")
+ #this now updates primary F as well
# response = raw_input("updated dvo ")
@@ -725,5 +729,5 @@
# now update StackMeta with correct number of inputs
- sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)"
+ sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage)"
self.scratchDb.execute(sql)
@@ -940,4 +944,5 @@
a.objID = b.objID, \
a.infoFlag = b.flags << 45 | a.infoFlag \
+ a.primaryF = ((b.flags & 0x10000) >> 16 ) \
WHERE a.ippDetectID = b.ippDetectID \
AND b.imageID = " + str(imageID)
Index: /branches/eam_branches/ipp-20130509/ppBackground/src/ppBackgroundArguments.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppBackground/src/ppBackgroundArguments.c (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppBackground/src/ppBackgroundArguments.c (revision 35748)
@@ -69,6 +69,6 @@
if (!data->patternName && !data->backgroundName && !data->auxMaskName) {
- psError(PPBACKGROUND_ERR_CONFIG, true, "Must specify at least one of -pattern and -background -auxmask");
- return false;
+// psError(PPBACKGROUND_ERR_CONFIG, true, "Must specify at least one of -pattern and -background -auxmask");
+// return false;
}
if (!data->imageName) {
Index: /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeLoop.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeLoop.c (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeLoop.c (revision 35748)
@@ -355,5 +355,6 @@
// call: pmDarkCombine(outCell, fileGroup->readouts, iter, rej, maskVal);
if (!psThreadJobAddPending(job)) {
- goto ERROR;
+ // goto ERROR;
+ // continue; // I don't care.
}
break;
Index: /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeReadChunk.c
===================================================================
--- /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeReadChunk.c (revision 35747)
+++ /branches/eam_branches/ipp-20130509/ppMerge/src/ppMergeReadChunk.c (revision 35748)
@@ -24,5 +24,5 @@
"INPUTS.VARIANCES");// Do we have variances?
int rows = psMetadataLookupS32(NULL, config->arguments, "ROWS"); // Number of rows to read per chunk
-
+ int rejectedInputs = 0;
// select an available fileGroup
while (1) {
@@ -73,4 +73,5 @@
psWarning ("skipping video cell (1: %ld)", (long) pthread_self());
inRO->process = false;
+ rejectedInputs++;
continue;
}
@@ -86,4 +87,5 @@
psWarning ("skipping video cell (2: %ld)", (long) pthread_self());
inRO->process = false;
+ rejectedInputs++;
continue;
}
@@ -94,4 +96,5 @@
psWarning ("skipping video cell (3: %ld)", (long) pthread_self());
inRO->process = false;
+ rejectedInputs++;
continue;
}
@@ -119,12 +122,20 @@
}
}
-
+
// fprintf (stderr, "%lx, %lx, %lx\n", (long int) inRO->image, (long int) inRO->mask, (long int) inRO->variance);
-
- if (!keepReading) {
- return NULL;
- }
+ if (!keepReading) {
+ return NULL;
+ }
+
+
}
-
+ psLogMsg("ppMerge",PS_LOG_INFO,"%d %ld %d %d %d",
+ j,readouts->n,rejectedInputs,fileGroup->read,fileGroup->busy);
+ if (rejectedInputs >= readouts->n) {
+ fileGroup->read = false;
+ fileGroup->busy = false;
+ return NULL;
+ }
+
fileGroup->read = fileGroup->busy = true;
return fileGroup;