Index: branches/eam_branches/ipp-20151113/ippToPsps/config/bundle.pl =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/config/bundle.pl (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/config/bundle.pl (revision 39224) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +system("rm -f *.csv"); +my @tables = ; +foreach my $t (@tables) { + my $csv = $t; + $csv =~ s/vot$/csv/; + system("./parse_vot.pl $t > $csv"); +} + +system("./parse_schema_browser.pl schema_browser.vot"); +system("./render.pl"); +system("pdflatex schema.tex"); + + +my $svn = ''; +open(SI, "svn info . | "); +while () { + chomp; + if ($_ =~ /Last Changed Rev:/) { + $svn = (split /\s+/, $_)[3]; + } +} +close(SI); + +$svn = 'r' . $svn; + +system("mv schema.pdf schema.${svn}.pdf"); +system("tar -zcf schema.${svn}.tgz schema.${svn}.pdf tables.*.csv SchemaTables.csv SchemaTableType.csv"); Index: branches/eam_branches/ipp-20151113/ippToPsps/config/schema_browser.vot =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/config/schema_browser.vot (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/config/schema_browser.vot (revision 39224) @@ -123,4 +123,10 @@
+ + +
+
+ + Index: branches/eam_branches/ipp-20151113/ippToPsps/config/tables.IN.vot =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/config/tables.IN.vot (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/config/tables.IN.vot (revision 39224) @@ -862,5 +862,5 @@ - Contains information flag values that define ForcedGalaxyShape chisq surface fit failuers. + Contains information flag values that define ForcedGalaxyShape chisq surface fit failures. Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/batch.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/batch.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/batch.py (revision 39224) @@ -104,5 +104,5 @@ # create DVO tables (scratch database) - self.scratchDb.createDvoTables() + ### XXX self.scratchDb.createDvoTables() <-- remove this # dump stuff to the log Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/cleanup.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/cleanup.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/cleanup.py (revision 39224) @@ -50,4 +50,5 @@ self.clean("FW") self.clean("FO") + self.clean("FG") self.checkClientStatus() Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/diffbatch.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/diffbatch.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/diffbatch.py (revision 39224) @@ -141,5 +141,5 @@ # look up the path base once - pathBase = gpc1Db.getDiffStagePathBase(skychunk.dvoLabel, int(diffSkyFileID)) + pathBase = gpc1Db.getDiffStagePathBase(int(diffSkyFileID)) # select the two (possible) cmf files (.cmf and .inv.cmf) @@ -336,5 +336,5 @@ else: fluxScaleStr = "-1" - detIDoffset = self.posImageDetCount + detIDoffset = self.posImageDetCount[num] if isPosFile or self.haveNegFits[num]: @@ -723,5 +723,5 @@ self.haveSrcPos = {} self.haveNegFits = {} - self.posImageDetCount = 0 + self.posImageDetCount = {} for num in self.number: @@ -742,4 +742,5 @@ ## if we have a negative image (inv.cmf), load that as well: self.haveNegFits[num] = 0 + self.posImageDetCount[num] = 0 if not self.negFits[num] == None: fileNameNeg = self.negFits[num].getPath() @@ -759,5 +760,6 @@ posTableName = "SkyChip_psf_" + str(num) - self.posImageDetCount = self.scratchDb.getRowCount(posTableName) + # self.posImageDetCount[num] = self.scratchDb.getMaxIPP_IDET(posTableName) + self.posImageDetCount[num] = self.scratchDb.getRowCount(posTableName) ## use stilts to get a list of the tables from the header Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/dvo.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/dvo.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/dvo.py (revision 39224) @@ -72,5 +72,5 @@ # response = raw_input("(y/n) ") # if response == "y": - self.resetAllTables() + self.resetAllTables() ## called if we need to change DVOs ''' @@ -95,4 +95,10 @@ self.correctDvo = True if not self.loadSkyTable(): + self.correctDvo = False + return + if not self.loadPhotcodes(): + self.correctDvo = False + return + if not self.loadImages(): self.correctDvo = False return @@ -298,6 +304,6 @@ return False - # update the two meta tables, Images and SkyTable - self.loadSkyTable() + # update the SkyTable + self.loadSkyTable() ## in setSkyArea # set bounding-box coords @@ -508,36 +514,39 @@ def sync(self): - # go no further if we've already partly ingested a different DVO - if not self.correctDvo: return False - - # check we have something to do - if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1: - self.logger.infoPair("Nothing to ingest or purge", "finished ingest") - return True - - # make sure we have an up-to-date Images table - self.loadImages() - - # purge stuff - self.purgeRegions(self.regionsToPurge) - - totalRegions = len(self.regionsToIngest) - if totalRegions > 0: self.logger.infoSeparator() - - # now loop through all regions and ingest to MySQL - ingestedRegionsCount = 0 - regionCount = 1 - for region in self.regionsToIngest: - - self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region)) - if self.ingestRegion(region): ingestedRegionsCount += 1 - regionCount += 1 - - self.logger.infoSeparator() - self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest))) - - # update lists after attempted sync - self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec) - self.printSummary() + print "---- sync is disabled ----" + os._exit(3) + + ### XXX # go no further if we've already partly ingested a different DVO + ### XXX if not self.correctDvo: return False + ### XXX + ### XXX # check we have something to do + ### XXX if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1: + ### XXX self.logger.infoPair("Nothing to ingest or purge", "finished ingest") + ### XXX return True + ### XXX + ### XXX # make sure we have an up-to-date Images table + ### XXX self.loadImages() + ### XXX + ### XXX # purge stuff + ### XXX self.purgeRegions(self.regionsToPurge) + ### XXX + ### XXX totalRegions = len(self.regionsToIngest) + ### XXX if totalRegions > 0: self.logger.infoSeparator() + ### XXX + ### XXX # now loop through all regions and ingest to MySQL + ### XXX ingestedRegionsCount = 0 + ### XXX regionCount = 1 + ### XXX for region in self.regionsToIngest: + ### XXX + ### XXX self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region)) + ### XXX if self.ingestRegion(region): ingestedRegionsCount += 1 + ### XXX regionCount += 1 + ### XXX + ### XXX self.logger.infoSeparator() + ### XXX self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest))) + ### XXX + ### XXX # update lists after attempted sync + ### XXX self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec) + ### XXX self.printSummary() return True @@ -667,5 +676,5 @@ includes purging detections outside sky area ''' - def nativeIngestDetections(self, boxId, boxDim): + def nativeIngestDetections(self, boxId, boxDim, myBatchType): # clear the 'ingested' field for all boxes owned by this host @@ -688,5 +697,5 @@ # make sure we have an up-to-date Images table - self.loadImages() + self.loadImages() # this is in nativeIngestDetections (remove?) # the box dimensions are the area used to select the items of @@ -695,17 +704,16 @@ # to grab all detections / objects which are contributed by an # item with the center in the given box - dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC'])) - dD_border = self.skychunk.BORDER - - #This is a terrible hack but I'm tired of it taking a decade to ingest the FW skycells. - - #if self.batchType == "ST": - # dR_border = .4 / math.cos(math.radians(boxDim['DEC'])) - # dD_border = .4 - #if self.batchType == "FW": - # dR_border = .4 / math.cos(math.radians(boxDim['DEC'])) - # dD_border = .4 - - + + # P2 and DF need to use a full GPC1 exposure radius: + myBorder = self.skychunk.BORDER_GPC1 + + # ST and FW need to use just a skycell + if myBatchType == "ST" : + myBorder = self.skychunk.BORDER_SKYCELL + if myBatchType == "FW" : + myBorder = self.skychunk.BORDER_SKYCELL + + dR_border = myBorder / math.cos(math.radians(boxDim['DEC'])) + dD_border = myBorder minRA = boxDim['minRA'] - dR_border @@ -788,9 +796,8 @@ else: cmd += " -photcode-start 12000 -photcode-end 12500" - ## if (useFW == 1): - ## if (self.config.camera == "simtest"): - ## print "need photcodes for non gpc1 camera" - ## else: - ## print "need photcode for warp for gpc1 camera, default is no selection, should work, could be slow" + + if (useST + useP2 + useDF + useFW > 1): + self.logger.errorPair("multiple batch types in a run", "NOW ILLEGAL") + os._exit(5) ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding? Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/dvoobjects.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/dvoobjects.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/dvoobjects.py (revision 39224) @@ -28,5 +28,5 @@ # declare DVO file types of interest self.ingestFileTypes = ['cpt', 'cps'] - self.loadPhotcodes() + self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches ''' Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/forcedgalaxybatch.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/forcedgalaxybatch.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/forcedgalaxybatch.py (revision 39224) @@ -72,4 +72,17 @@ ''' + Generate ippObjID and make it an index + ''' + def addIppObjID (self, tableName): + sql = "ALTER TABLE " + tableName + " ADD COLUMN ippObjID BIGINT" + self.scratchDb.execute(sql) + + sql = "CREATE INDEX ippObjID_index ON " + tableName + " ( ippObjID ) "; + self.scratchDb.execute(sql) + + sql = "UPDATE " + tableName + " set ippObjID = (OBJ_ID + (CAT_ID << 32))" + self.scratchDb.execute(sql) + + ''' Overriden from batch base class to import directly from DVO for forced galaxy shape parameters ''' @@ -101,4 +114,9 @@ self.logger.infoPair("0 rows for ","cpq table") return False + + # add ippObjID columns and index them: + self.addIppObjID (cptTableName) + self.addIppObjID (cpqTableName) + return True @@ -178,9 +196,21 @@ # sqlLine.group("a." + filter[1] + "GalDecErr", "b.DEC_ERR") - sql = sqlLine.makeEquals("WHERE a.ippObjID = (b.OBJ_ID + (b.CAT_ID << 32)) AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode)) + sql = sqlLine.makeEquals("WHERE a.ippObjID = b.ippObjID AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode)) self.logger.debugPair("cpq sql: ", sql) self.scratchDb.execute(sql) + + ## # check on the status as we go: + ## sql = "select count(*) as N, galModelType, nFilter, count(*)*nFilter as S from ForcedGalaxyShape group by galModelType, nFilter" + ## rs = self.scratchDb.executeQuery(sql) + ## + ## while rs.next(): + ## N = rs.getInt(1) + ## M = rs.getInt(2) + ## F = rs.getInt(3) + ## S = rs.getInt(4) + ## + ## print "N: %d, TYPE: %d, nFilter: %d, Sum: %d" % (N, M, F, S) ''' @@ -207,5 +237,5 @@ sqlLine.group("objID", "EXT_ID") - sqlLine.group("ippObjID", "OBJ_ID + (CAT_ID << 32)") + sqlLine.group("ippObjID", "ippObjID") sqlLine.group("surveyID", str(self.surveyID)) sqlLine.group("randomForcedGalID", "RAND("+str(self.batchID)+")") @@ -215,5 +245,5 @@ sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName) - print "--- sql: " + sql + # print "--- sql: " + sql self.scratchDb.execute(sql) Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/gpc1db.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/gpc1db.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/gpc1db.py (revision 39224) @@ -79,6 +79,6 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ - AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \ + AND decl >= RADIANS(" + str(minDec) + ") AND decl < RADIANS(" + str(maxDec) + ") \ + AND ra >= RADIANS(" + str(minRA) + ") AND ra < RADIANS(" + str(maxRA) + ") \ AND dateobs >= '" + tstart + "'\ AND dateobs <= '" + tend + "'" @@ -103,6 +103,6 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \ + AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \ AND mjd_obs >= (to_days('" + tstart + "')-678941) \ AND mjd_obs <= (to_days('" + tend + "') - 678941) " @@ -121,6 +121,5 @@ JOIN mergedvodbProcessed using (merge_id) \ JOIN diffInputSkyfile \ - ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \ - AND stage = 'diff') \ + ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \ JOIN skycell using (skycell_id, tess_id) \ JOIN warpRun on (warp1 = warp_id \ @@ -137,6 +136,6 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \ + AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \ AND dateobs >= '" + tstart + "'\ AND dateobs <= '" + tend + "'\ @@ -164,6 +163,6 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ - AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \ + AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \ AND mjd_obs >= (to_days('" + tstart + "')-678941) \ AND mjd_obs <= (to_days('" + tend + "') - 678941) \ @@ -308,6 +307,5 @@ JOIN mergedvodbProcessed using (merge_id) \ JOIN diffInputSkyfile \ - ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \ - AND stage = 'diff') \ + ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \ JOIN skycell using (skycell_id, tess_id) \ JOIN warpRun on (warp1 = warp_id \ @@ -513,12 +511,31 @@ get diff stage cmf name ''' - def getDiffStagePathBase(self, dvoDb, diffSkyfileID): + def getDiffStagePathBase(self, diffSkyfileID): + sql = "SELECT DISTINCT diffSkyfile.path_base \ + FROM diffInputSkyfile \ + JOIN diffSkyfile using (diff_id, skycell_id) \ + WHERE diff_skyfile_id = " + str(diffSkyfileID) + + rs = self.executeQuery(sql) + + if not rs.first(): + self.logger.errorPair("missing cmf entry diff_skyfile_id:", str(diff_skyfile_id)) + # self.logger.errorPair("no diff cmfs found for diff_skyfile_id = %d" %diff_skyfile_id) + return None + + pathBase = rs.getString(1) + return pathBase + + ''' + get diff stage cmf name + XXX EAM 20151119 : this function does too much work: you have a diffSkyfileID, that matches to a unique path_base + ''' + def getDiffStagePathBaseOldVersion(self, dvoDb, diffSkyfileID): print "got here, diffstagecmf" sql = "SELECT DISTINCT diffSkyfile.path_base \ FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \ JOIN addRun using (minidvodb_name) \ - JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \ - AND diff_id = stage_id AND stage = 'diff') \ - JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \ + JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 AND diff_id = stage_id AND stage = 'diff') \ + JOIN diffSkyfile using (diff_id, skycell_id) \ WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \ AND minidvodbRun.state = 'merged' \ @@ -561,4 +578,5 @@ ''' get diff stage cmf name + XXX EAM 20151119 This function is broken AND it does too much work ''' def getDiffStageCmf(self, dvoDb, diffSkyfileID): Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/ipptopspsdb.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/ipptopspsdb.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/ipptopspsdb.py (revision 39224) @@ -324,5 +324,5 @@ AND processed = 1 \ AND loaded_to_datastore = 1 \ - AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" + AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()" total = -1 @@ -421,5 +421,5 @@ AND timestamp > '" + self.skychunk.epoch + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ - AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" + AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()" try: @@ -545,5 +545,5 @@ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND batch_type != 'IN' \ - AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()" + AND timestamp >= now() - INTERVAL 12 HOUR AND timestamp < now()" try: @@ -1158,6 +1158,4 @@ self.skychunk.boxSize = rs.getDouble(10) - # self.skychunk.halfBox = self.skychunk.boxSize/2.0 - # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2) self.skychunk.basePath = rs.getString(11) @@ -1240,6 +1238,6 @@ sql = "SELECT id FROM box \ WHERE skychunk = '" + self.skychunk.name + "' \ - AND ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ - AND dec_center BETWEEN " + str(minDEC) + " AND " + str(maxDEC) + AND ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \ + AND dec_center >= " + str(minDEC) + " AND dec_center < " + str(maxDEC) # AND box_side = " + str(self.skychunk.boxSize) @@ -1424,5 +1422,5 @@ FROM box \ JOIN pending ON (id = box_id) \ - WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \ AND batch_type = '" + batchType + "' \ AND skychunk = '" + self.skychunk.name + "' \ @@ -1503,5 +1501,5 @@ FROM box \ JOIN pending ON (id = box_id) \ - WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ + WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \ AND skychunk = '" + self.skychunk.name + "' \ ORDER BY dec_center DESC" @@ -1712,6 +1710,6 @@ sql = "SELECT DISTINCT stage_id \ FROM all_pending \ - WHERE ra_bore BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \ - AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC) + WHERE ra_bore >= " + str(minRA) + " AND ra_bore < " + str(maxRA) + " \ + AND dec_bore >= " + str(minDEC) + " AND dec_bore < " + str(maxDEC) try: Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/loader.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/loader.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/loader.py (revision 39224) @@ -172,5 +172,5 @@ if (batchType == "P2" or batchType == "ST" or batchType =="DF" or batchType == "FW"): if self.config.resetDvo or not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost): - if not self.dvo.nativeIngestDetections(boxId, boxDim): + if not self.dvo.nativeIngestDetections(boxId, boxDim, batchType): self.logger.error("Unable to ingest detections with nativeIngest") return False Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/queue.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/queue.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/queue.py (revision 39224) @@ -146,4 +146,5 @@ # XXX not sure why this is not done with a for-loop... + # answer: because python does not have a good float-style for-loop while dec <= self.skychunk.maxDec: while ra <= self.skychunk.maxRa: @@ -158,5 +159,5 @@ ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC) - + self.logger.info("| %5.1f | %5.1f | %9d | %9d |" % ( ra, @@ -165,6 +166,11 @@ len(ids))) - if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids) - # print "inserted pending" + if len(ids) > 0: + ## for myID in ids: + ## print "id: %d : %10.6f - %10.6f | %10.6f - %10.6f" % (myID, minRA, maxRA, minDEC, maxDEC) + + self.ippToPspsDb.insertPending(box_id, batchType, ids) + # print "inserted pending" + ra = ra + 2*dR # print "new ra: ", ra Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/scratchdb.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/scratchdb.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/scratchdb.py (revision 39224) @@ -406,4 +406,18 @@ return size + + ''' + Returns a row count for this table + ''' + def getMaxIPP_IDET(self, table): + + sql = "SELECT MAX(IPP_IDET) FROM " + table + rs = self.executeQuery(sql) + + if not rs.first(): + self.logger.exception("Could not count rows for table: '" + table + "'") + return 0 + + return rs.getInt(1) ''' @@ -655,28 +669,32 @@ def resetAllDvoTables(self): - # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint - self.logger.infoPair("Dropping table", self.dvoImagesTable) - self.dropTable(self.dvoImagesTable) - self.logger.infoPair("Dropping table", self.dvoSkyTable) - self.dropTable(self.dvoSkyTable) - self.logger.infoPair("Dropping table", self.dvoDoneTable) - self.dropTable(self.dvoDoneTable) - - # blow away existing dvoDetection table & re-crate - if not self.resetDvoDetectionTable(): - self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull") - return False - - # create dvoDone table - self.logger.infoPair("Creating table", self.dvoDoneTable) - sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)" - try: self.execute(sql) - except: - self.logger.errorPair("Unable to create table", self.dvoDoneTable) - return False - - self.changeEngineToInnoDB(self.dvoDoneTable) - - return True + # re-create the definition tables + self.createDvoTables() + + # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint + self.logger.infoPair("Dropping table", self.dvoImagesTable) + self.dropTable(self.dvoImagesTable) + self.logger.infoPair("Dropping table", self.dvoSkyTable) + self.dropTable(self.dvoSkyTable) + self.logger.infoPair("Dropping table", self.dvoDoneTable) + self.dropTable(self.dvoDoneTable) + + # blow away existing dvoDetection table & re-crate + if not self.resetDvoDetectionTable(): + self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull") + return False + + # create dvoDone table + self.logger.infoPair("Creating table", self.dvoDoneTable) + sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)" + try: + self.execute(sql) + except: + self.logger.errorPair("Unable to create table", self.dvoDoneTable) + return False + + self.changeEngineToInnoDB(self.dvoDoneTable) + + return True ''' Index: branches/eam_branches/ipp-20151113/ippToPsps/jython/skychunk.py =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/jython/skychunk.py (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/jython/skychunk.py (revision 39224) @@ -39,5 +39,6 @@ # XXX this is a poor place to put this information -- it is completely gpc-specific. # probably should go elsewhere. This number is scaled in loader.py by cos(dec) - self.BORDER = 1.65 + self.BORDER_GPC1 = 1.65 + self.BORDER_SKYCELL = 0.40 self.isLoaded = False Index: branches/eam_branches/ipp-20151113/ippToPsps/test/fulltest.sh =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/fulltest.sh (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/fulltest.sh (revision 39224) @@ -561,4 +561,5 @@ rm -rf $OUTDIR/FO + rm -rf $OUTDIR/FG ippjython loader.py $options $forcedobjqueuename once -simtest < /dev/null Index: branches/eam_branches/ipp-20151113/ippToPsps/test/gpc1schema.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/gpc1schema.sql (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/gpc1schema.sql (revision 39224) @@ -21,5 +21,5 @@ CREATE TABLE diffRun ( diff_id INT, tess_id CHAR(64), state CHAR(16), diff_mode INT, software_ver CHAR(16) ); CREATE TABLE diffInputSkyfile ( diff_id INT, warp1 INT, stack2 INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64) ); - CREATE TABLE diffSkyfile ( diff_id INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64), path_base CHAR(255) ); + CREATE TABLE diffSkyfile ( diff_id INT, skycell_id CHAR(64), path_base CHAR(255) ); CREATE TABLE addRun ( add_id INT, stage CHAR(64), stage_extra1 INT, state CHAR(64), stage_id INT, minidvodb_name CHAR(64) ); Index: branches/eam_branches/ipp-20151113/ippToPsps/test/mkgpc1data.dvo =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/mkgpc1data.dvo (revision 39135) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/mkgpc1data.dvo (revision 39224) @@ -654,5 +654,5 @@ dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M") dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID, $SKYCELL ) - dbinsert diffSkyfile (diff_id, diff_skyfile_id, tess_id, skycell_id, path_base) values ($difID, $diffSkyfileID, $TESS_ID, $SKYCELL, "$filename") + dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, $SKYCELL, "$filename") end Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v0.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v0.sql (revision 39135) +++ (revision ) @@ -1,25 +1,0 @@ - --- get a set of measurements matched by detID -select - gra, - gdec, - objID, - HEX(ippObjID), - surveyID, - tessID, - projectionID, - skyCellID, - randomStackObjID, - stackDetectRowID, - primaryDetection, --- bestDetection, - dvoRegionID, - dataRelease, - gippDetectID, - gstackDetectID, - gstackMetaID, - HEX(ginfoFlag), - HEX(ginfoFlag2), - gnFrames -from StackObjectThin -limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v1.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v1.sql (revision 39135) +++ (revision ) @@ -1,44 +1,0 @@ --- compare g mags across stack tables -select - gra, - gdec, - graErr, - gdecErr, - gPSFMag, - gKronMag, - gApMag, - gExpMag, - gSerMag, - gDeVMag, - gpetMag, - gPSFMagErr, - gKronMagErr, - gApMagErr, - gExpMagErr, - gSerMagErr, - gDeVMagErr, - gpetMagErr, - gpsfLikelihood, - gpsfQf, - gpsfQfPerfect, - gippDetectID - from StackObjectThin - -- left join StackObjectRaw using (gippDetectID) - -- left join StackModelFitExp using (gippDetectID) - -- left join StackModelFitDeV using (gippDetectID) - -- left join StackModelFitSer using (gippDetectID) - -- left join StackPetrosian using (gippDetectID) - -- left join StackApFlxFull using (gippDetectID) - -- left join StackApFlxFullC1 using (gippDetectID) - -- left join StackApFlxFullC2 using (gippDetectID) - join StackObjectRaw using (gippDetectID) - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - join StackPetrosian using (gippDetectID) - join StackApFlxFull using (gippDetectID) - join StackApFlxFullC1 using (gippDetectID) - join StackApFlxFullC2 using (gippDetectID) - -- where (gippDetectID = 189) - -- where (gpsfQfPerfect > 0.85) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v2.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v2.sql (revision 39135) +++ (revision ) @@ -1,29 +1,0 @@ --- compare g raw PSF values -select - gippDetectID, - gxPos, - gyPos, - gxPosErr, - gyPosErr, - gpsfMajorFWHM, - gpsfMinorFWHM, - gpsfTheta, - gpsfCore, - gpsfLikelihood, - gpsfQf, - gpsfQfPerfect, - gpsfChiSq, - gmomentXX, - gmomentXY, - gmomentYY, - gmomentR1, - gmomentRH, - gApFillFac, - gKronRad, - gsky, - gskyErr, - gexpTime, - gzp - from StackObjectThin - join StackObjectRaw using (gippDetectID) - limit 80 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v3.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v3.sql (revision 39135) +++ (revision ) @@ -1,32 +1,0 @@ --- compare g fluxes across tables -select - gra, - gdec, - gPSFFlux, - gKronFlux, - gApFlux, - gPSFFluxErr, - gKronFluxErr, - gApFluxErr, - gApFillFac, - gflxR1, - gflxR2, - gflxR3, - gflxR4, - gflxR5, - gflxR6, - gflxR7, - gflxR8, - gflxR9, - gflxR10, - gippDetectID - from StackObjectThin - join StackObjectRaw using (gippDetectID) - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - join StackPetrosian using (gippDetectID) - join StackApFlxFull using (gippDetectID) - join StackApFlxFullC1 using (gippDetectID) - join StackApFlxFullC2 using (gippDetectID) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v4.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v4.sql (revision 39135) +++ (revision ) @@ -1,38 +1,0 @@ --- compare g fluxes across tables -select - gra, - gdec, - gPSFFlux, - gKronFlux, - gApFlux, - gflxR1, - gc1flxR1, - gc2flxR1, - gflxR2, - gc1flxR2, - gc2flxR2, - gflxR3, - gc1flxR3, - gc2flxR3, - gflxR4, - gc1flxR4, - gc2flxR4, - gflxR5, - gc1flxR5, - gc2flxR5, - gPSFFluxErr, - gKronFluxErr, - gApFluxErr, - gApFillFac, - gpsfQfPerfect, - gippDetectID - from StackObjectThin - join StackObjectRaw using (gippDetectID) - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - join StackPetrosian using (gippDetectID) - join StackApFlxFull using (gippDetectID) - join StackApFlxFullC1 using (gippDetectID) - join StackApFlxFullC2 using (gippDetectID) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v5.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v5.sql (revision 39135) +++ (revision ) @@ -1,32 +1,0 @@ --- compare g mags across stack tables -select - gra, - gdec, - gExpMag, - gSerMag, - gDeVMag, - gExpRadius, - gSerRadius, - gDeVRadius, - gExpAb, - gSerAb, - gDeVAb, - gExpPhi, - gSerPhi, - gDeVPhi, - gippDetectID - from StackObjectThin - -- left join StackObjectRaw using (gippDetectID) - -- left join StackModelFitExp using (gippDetectID) - -- left join StackModelFitDeV using (gippDetectID) - -- left join StackModelFitSer using (gippDetectID) - -- left join StackPetrosian using (gippDetectID) - -- left join StackApFlxFull using (gippDetectID) - -- left join StackApFlxFullC1 using (gippDetectID) - -- left join StackApFlxFullC2 using (gippDetectID) - join StackObjectRaw using (gippDetectID) - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - -- where (gpsfQfPerfect > 0.85) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v6.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v6.sql (revision 39135) +++ (revision ) @@ -1,25 +1,0 @@ --- compare g mags across stack tables -select - gra, - gdec, - gExpMag, - gExpMagErr, - gExpRadius, - gExpRadiusErr, - gExpAb, - gExpAbErr, - gExpPhi, - gExpPhiErr, - gExpRa, - gExpDec, - gExpRaErr, - gExpDecErr, - gExpCf, - gExpLikelihood, - gippDetectID - from StackObjectThin - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - -- where (gpsfQfPerfect > 0.85) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v7.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v7.sql (revision 39135) +++ (revision ) @@ -1,25 +1,0 @@ --- compare g mags across stack tables -select - gra, - gdec, - gDeVMag, - gDeVMagErr, - gDeVRadius, - gDeVRadiusErr, - gDeVAb, - gDeVAbErr, - gDeVPhi, - gDeVPhiErr, - gDeVRa, - gDeVDec, - gDeVRaErr, - gDeVDecErr, - gDeVCf, - gDeVLikelihood, - gippDetectID - from StackObjectThin - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - -- where (gpsfQfPerfect > 0.85) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v8.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v8.sql (revision 39135) +++ (revision ) @@ -1,27 +1,0 @@ --- compare g mags across stack tables -select - gra, - gdec, - gSerMag, - gSerMagErr, - gSerRadius, - gSerRadiusErr, - gSerAb, - gSerAbErr, - gSerNu, - gSerNuErr, - gSerPhi, - gSerPhiErr, - gSerRa, - gSerDec, - gSerRaErr, - gSerDecErr, - gSerCf, - gSerLikelihood, - gippDetectID - from StackObjectThin - join StackModelFitExp using (gippDetectID) - join StackModelFitDeV using (gippDetectID) - join StackModelFitSer using (gippDetectID) - -- where (gpsfQfPerfect > 0.85) - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v9.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/sascheck.v9.sql (revision 39135) +++ (revision ) @@ -1,29 +1,0 @@ --- compare g fluxes across tables -select - gra, - gdec, - gxPos, - gyPos, - gPSFFlux, - gApFlux, - gPSFFluxErr, - gApFluxErr, - gApFillFac, - gflxR3, - gflxR4, - gflxR5, - gflxR3Err, - gflxR4Err, - gflxR5Err, - gflxR3Std, - gflxR4Std, - gflxR5Std, - gflxR3Fill, - gflxR4Fill, - gflxR5Fill, - gippDetectID - from StackObjectThin - join StackObjectRaw using (gippDetectID) - join StackApFlx using (gippDetectID) - where gPSFFlux > 1000 - limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/force.sh =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/force.sh (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/force.sh (revision 39224) @@ -0,0 +1,26 @@ +#!/bin/csh + +set catdir = catdir.test + +mkcmf -forcedwarp -photcode SIMTEST.g.ForcedWarp testdata/test.01.txt testdata/warp.01.cmf -extroot SkyChip -date 2010/01/01 -time 01:00:00 -mjd 55197.0416667 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 1 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.g.ForcedWarp testdata/test.02.txt testdata/warp.02.cmf -extroot SkyChip -date 2010/01/01 -time 02:00:00 -mjd 55197.0833333 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 2 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.g.ForcedWarp testdata/test.03.txt testdata/warp.03.cmf -extroot SkyChip -date 2010/01/01 -time 03:00:00 -mjd 55197.125 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 3 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.g.ForcedWarp testdata/test.04.txt testdata/warp.04.cmf -extroot SkyChip -date 2010/01/01 -time 04:00:00 -mjd 55197.1666667 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 4 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp testdata/test.13.txt testdata/warp.13.cmf -extroot SkyChip -date 2010/01/02 -time 01:00:00 -mjd 55198.0416667 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 13 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp testdata/test.14.txt testdata/warp.14.cmf -extroot SkyChip -date 2010/01/02 -time 02:00:00 -mjd 55198.0833333 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 14 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp testdata/test.15.txt testdata/warp.15.cmf -extroot SkyChip -date 2010/01/02 -time 03:00:00 -mjd 55198.125 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 15 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedwarp -photcode SIMTEST.r.ForcedWarp testdata/test.16.txt testdata/warp.16.cmf -extroot SkyChip -date 2010/01/02 -time 04:00:00 -mjd 55198.1666667 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 16 -sourceID 2 -tess_id RINGS.V3 -skycell skycell.1133.081 -bad-psfqf-frac 0.2 +mkcmf -forcedgalaxy -photcode SIMTEST.g.ForcedWarp testdata/test.01.txt testdata/fgal.01.cmf -extroot SkyChip -date 2008/1/1 -time 0:00:00 -mjd 54466.0 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 1 -sourceID 1 -tess_id RINGS.V3 -skycell skycell.1133.081 +mkcmf -forcedgalaxy -photcode SIMTEST.r.ForcedWarp testdata/test.04.txt testdata/fgal.04.cmf -extroot SkyChip -date 2008/1/1 -time 3:00:00 -mjd 54466.0 -radec 10.0 20.0 -type PS1_V5_Lensing -imageID 4 -sourceID 1 -tess_id RINGS.V3 -skycell skycell.1133.081 + +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.01.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.02.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.03.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.04.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.13.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.14.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.15.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad +addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest testdata/warp.16.cmf -D CATFORMAT PS1_V5 -quick-airmass -xrad + +loadgalphot -D CATDIR $catdir testdata/fgal.01.cmf +loadgalphot -D CATDIR $catdir testdata/fgal.04.cmf Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/input.gdb =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/input.gdb (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/input.gdb (revision 39224) @@ -0,0 +1,77 @@ + +define showdata + set $i = 0 + while $i < Npts + printf "%6.3f %6.3f %10.6f %d\n", Xpt[$i], Ypt[$i], chisq[$i], mask[$i] + set $i = $i + 1 + end +end + +define showlg + set $ix = 0 + while $ix < fit->mterm + set $iy = 0 + while $iy < fit->mterm + printf "%6.3f \n", fit->s[$ix][$iy] + set $iy = $iy + 1 + end + set $ix = $ix + 1 + printf "\n" + end +end + +define showmn + set $ix = 0 + while $ix < mterm + set $iy = 0 + while $iy < mterm + printf "%6.3f \n", s[$ix][$iy] + set $iy = $iy + 1 + end + set $ix = $ix + 1 + printf "\n" + end +end + +define showlB + set $iy = 0 + while $iy < fit->wterm + printf "%6.3f \n", fit->b[$iy][0] + set $iy = $iy + 1 + end +end + +define showmB + set $iy = 0 + while $iy < wterm + printf "%6.3f \n", b[$iy][0] + set $iy = $iy + 1 + end +end + +define showlc + set $ix = 0 + while $ix < fit->wterm + set $iy = 0 + while $iy < fit->wterm + printf "%6.3f ", fit->c[$ix][$iy] + set $iy = $iy + 1 + end + set $ix = $ix + 1 + printf "\n" + end +end + +define showmc + set $ix = 0 + while $ix < wterm + set $iy = 0 + while $iy < wterm + printf "%6.3f ", c[$ix][$iy] + set $iy = $iy + 1 + end + set $ix = $ix + 1 + printf "\n" + end +end + Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/mana.sh =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/mana.sh (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/mana.sh (revision 39224) @@ -0,0 +1,87 @@ + +if (not($?VERBOSE)) set VERBOSE = 0 +if (not($?SHOW_PLOTS)) set SHOW_PLOTS = 1 + +macro checkfit + if ($0 != 3) + echo "USAGE: checkfit (file) (number)" + break + end + + data $1 + read -fits SkyChip.xgal FR_MINOR_MIN FR_MINOR_MAX FR_MINOR_DEL FR_MAJOR_MIN FR_MAJOR_MAX FR_MAJOR_DEL GAL_CHISQ GAL_FLUX NPIX + + delete -q Xi Yi Xv Yv Zv + + local ix iy + + $N = 0 + for ix 0 5 + for iy 0 5 + $Vx = FR_MAJOR_MIN[$2] + FR_MAJOR_DEL[$2]*$ix + $Vy = FR_MINOR_MIN[$2] + FR_MINOR_DEL[$2]*$iy + + concat $ix Xi + concat $iy Yi + concat $Vx Xv + concat $Vy Yv + + # echo concat GAL_CHISQ:$N[0] Zv + concat GAL_CHISQ:$N[$2] Zv + if ($VERBOSE) + fprintf "%1d %1d %5.3f %5.3f %7.3f" Xi[$N] Yi[$N] Xv[$N] Yv[$N] Zv[$N] + end + $N ++ + end + end + + vstat -q Zv + set Zv = Zv * NPIX[$2] / $MIN + + if ($VERBOSE) + fit2d Xv Yv Zv 2 -clip 5 3 + else + fit2d Xv Yv Zv 2 -clip 5 3 -q + end + + $DET = 1 / (4*$CX0Y2*$CX2Y0 - $CX1Y1^2) + $Xm = ($CX1Y1*$CX0Y1 - 2*$CX0Y2*$CX1Y0) * $DET + $Ym = ($CX1Y1*$CX1Y0 - 2*$CX2Y0*$CX0Y1) * $DET + # echo $Xm $Ym + + applyfit2d Xv Yv Zvf + set dZ = Zv - Zvf + + if ($SHOW_PLOTS) + lim -n 0 Xv Yv; clear; box -ticks 1111 + vstat -q Zv + $Range = $MAX - $MIN + $Midpt = 0.5*($MAX + $MIN) + zplot -pt 2 Xv Yv Zv {$Midpt - 0.55*$Range} {$Midpt - 0.3*$Range} -c blue + zplot -pt 7 Xv Yv Zvf {$Midpt - 0.55*$Range} {$Midpt - 0.3*$Range} -c red + + dot $Xm $Ym -pt 7 -sz 3 -c red + + lim -n 1 Xv Yv; clear; box -ticks 1111 + vstat -q dZ + $Range = $MAX - $MIN + $Midpt = 0.5*($MAX + $MIN) + zplot -pt 10 Xv Yv dZ {$Midpt - 0.8*$Range} {$Midpt + 0.6*$Range} + + dot $Xm $Ym -pt 7 -sz 3 -c red + end + + $A = $CX2Y0 + $B = $CX1Y0 + 2*$CX2Y0*$Xm + $CX1Y1*$Ym + $C = -1 + + $dXm = (sqrt($B^2 - 4*$A*$C) - $B) / (2*$A) + + $A = $CX0Y2 + $B = $CX0Y1 + 2*$CX0Y2*$Ym + $CX1Y1*$Xm + $C = -1 + + $dYm = (sqrt($B^2 - 4*$A*$C) - $B) / (2*$A) + + echo ($Xm, $Ym) +/- ($dXm, $dYm) +end Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v0.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v0.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v0.sql (revision 39224) @@ -0,0 +1,25 @@ + +-- get a set of measurements matched by detID +select + gra, + gdec, + objID, + HEX(ippObjID), + surveyID, + tessID, + projectionID, + skyCellID, + randomStackObjID, + stackDetectRowID, + primaryDetection, +-- bestDetection, + dvoRegionID, + dataRelease, + gippDetectID, + gstackDetectID, + gstackMetaID, + HEX(ginfoFlag), + HEX(ginfoFlag2), + gnFrames +from StackObjectThin +limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v1.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v1.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v1.sql (revision 39224) @@ -0,0 +1,44 @@ +-- compare g mags across stack tables +select + gra, + gdec, + graErr, + gdecErr, + gPSFMag, + gKronMag, + gApMag, + gExpMag, + gSerMag, + gDeVMag, + gpetMag, + gPSFMagErr, + gKronMagErr, + gApMagErr, + gExpMagErr, + gSerMagErr, + gDeVMagErr, + gpetMagErr, + gpsfLikelihood, + gpsfQf, + gpsfQfPerfect, + gippDetectID + from StackObjectThin + -- left join StackObjectRaw using (gippDetectID) + -- left join StackModelFitExp using (gippDetectID) + -- left join StackModelFitDeV using (gippDetectID) + -- left join StackModelFitSer using (gippDetectID) + -- left join StackPetrosian using (gippDetectID) + -- left join StackApFlxFull using (gippDetectID) + -- left join StackApFlxFullC1 using (gippDetectID) + -- left join StackApFlxFullC2 using (gippDetectID) + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + -- where (gippDetectID = 189) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v2.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v2.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v2.sql (revision 39224) @@ -0,0 +1,29 @@ +-- compare g raw PSF values +select + gippDetectID, + gxPos, + gyPos, + gxPosErr, + gyPosErr, + gpsfMajorFWHM, + gpsfMinorFWHM, + gpsfTheta, + gpsfCore, + gpsfLikelihood, + gpsfQf, + gpsfQfPerfect, + gpsfChiSq, + gmomentXX, + gmomentXY, + gmomentYY, + gmomentR1, + gmomentRH, + gApFillFac, + gKronRad, + gsky, + gskyErr, + gexpTime, + gzp + from StackObjectThin + join StackObjectRaw using (gippDetectID) + limit 80 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v3.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v3.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v3.sql (revision 39224) @@ -0,0 +1,32 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFFlux, + gKronFlux, + gApFlux, + gPSFFluxErr, + gKronFluxErr, + gApFluxErr, + gApFillFac, + gflxR1, + gflxR2, + gflxR3, + gflxR4, + gflxR5, + gflxR6, + gflxR7, + gflxR8, + gflxR9, + gflxR10, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v4.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v4.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v4.sql (revision 39224) @@ -0,0 +1,38 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gPSFFlux, + gKronFlux, + gApFlux, + gflxR1, + gc1flxR1, + gc2flxR1, + gflxR2, + gc1flxR2, + gc2flxR2, + gflxR3, + gc1flxR3, + gc2flxR3, + gflxR4, + gc1flxR4, + gc2flxR4, + gflxR5, + gc1flxR5, + gc2flxR5, + gPSFFluxErr, + gKronFluxErr, + gApFluxErr, + gApFillFac, + gpsfQfPerfect, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + join StackPetrosian using (gippDetectID) + join StackApFlxFull using (gippDetectID) + join StackApFlxFullC1 using (gippDetectID) + join StackApFlxFullC2 using (gippDetectID) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v5.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v5.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v5.sql (revision 39224) @@ -0,0 +1,32 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gExpMag, + gSerMag, + gDeVMag, + gExpRadius, + gSerRadius, + gDeVRadius, + gExpAb, + gSerAb, + gDeVAb, + gExpPhi, + gSerPhi, + gDeVPhi, + gippDetectID + from StackObjectThin + -- left join StackObjectRaw using (gippDetectID) + -- left join StackModelFitExp using (gippDetectID) + -- left join StackModelFitDeV using (gippDetectID) + -- left join StackModelFitSer using (gippDetectID) + -- left join StackPetrosian using (gippDetectID) + -- left join StackApFlxFull using (gippDetectID) + -- left join StackApFlxFullC1 using (gippDetectID) + -- left join StackApFlxFullC2 using (gippDetectID) + join StackObjectRaw using (gippDetectID) + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v6.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v6.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v6.sql (revision 39224) @@ -0,0 +1,25 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gExpMag, + gExpMagErr, + gExpRadius, + gExpRadiusErr, + gExpAb, + gExpAbErr, + gExpPhi, + gExpPhiErr, + gExpRa, + gExpDec, + gExpRaErr, + gExpDecErr, + gExpCf, + gExpLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v7.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v7.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v7.sql (revision 39224) @@ -0,0 +1,25 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gDeVMag, + gDeVMagErr, + gDeVRadius, + gDeVRadiusErr, + gDeVAb, + gDeVAbErr, + gDeVPhi, + gDeVPhiErr, + gDeVRa, + gDeVDec, + gDeVRaErr, + gDeVDecErr, + gDeVCf, + gDeVLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v8.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v8.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v8.sql (revision 39224) @@ -0,0 +1,27 @@ +-- compare g mags across stack tables +select + gra, + gdec, + gSerMag, + gSerMagErr, + gSerRadius, + gSerRadiusErr, + gSerAb, + gSerAbErr, + gSerNu, + gSerNuErr, + gSerPhi, + gSerPhiErr, + gSerRa, + gSerDec, + gSerRaErr, + gSerDecErr, + gSerCf, + gSerLikelihood, + gippDetectID + from StackObjectThin + join StackModelFitExp using (gippDetectID) + join StackModelFitDeV using (gippDetectID) + join StackModelFitSer using (gippDetectID) + -- where (gpsfQfPerfect > 0.85) + limit 50 Index: branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v9.sql =================================================================== --- branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v9.sql (revision 39224) +++ branches/eam_branches/ipp-20151113/ippToPsps/test/testscripts/sascheck.v9.sql (revision 39224) @@ -0,0 +1,29 @@ +-- compare g fluxes across tables +select + gra, + gdec, + gxPos, + gyPos, + gPSFFlux, + gApFlux, + gPSFFluxErr, + gApFluxErr, + gApFillFac, + gflxR3, + gflxR4, + gflxR5, + gflxR3Err, + gflxR4Err, + gflxR5Err, + gflxR3Std, + gflxR4Std, + gflxR5Std, + gflxR3Fill, + gflxR4Fill, + gflxR5Fill, + gippDetectID + from StackObjectThin + join StackObjectRaw using (gippDetectID) + join StackApFlx using (gippDetectID) + where gPSFFlux > 1000 + limit 50