IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 3, 2015, 4:14:24 PM (11 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20151113
Files:
10 deleted
20 edited
2 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20151113

  • branches/eam_branches/ipp-20151113/ippToPsps

    • Property svn:ignore

      •  

        old new  
        1414install-sh
        1515Doxyfile
         16compile

  • branches/eam_branches/ipp-20151113/ippToPsps/config/schema_browser.vot

    r39082 r39224  
    123123        <TABLE name="ForcedMeanObject"> </TABLE>
    124124     </RESOURCE>
     125
     126     <RESOURCE name="ForcedGalaxyModelView">
     127        <TABLE name="ObjectThin"> </TABLE>
     128        <TABLE name="ForcedGalaxyShape"> </TABLE>
     129     </RESOURCE>
     130
    125131  </RESOURCE>
    126132

  • branches/eam_branches/ipp-20151113/ippToPsps/config/tables.IN.vot

    r39131 r39224  
    862862
    863863     <TABLE name="ForcedGalaxyShapeFlags">
    864       <DESCRIPTION>Contains information flag values that define ForcedGalaxyShape chisq surface fit failuers.  </DESCRIPTION>
     864      <DESCRIPTION>Contains information flag values that define ForcedGalaxyShape chisq surface fit failures.</DESCRIPTION>
    865865      <PARAM arraysize="1" datatype="char" ucd="meta.bib.author" name="Author" value="PSPS"></PARAM>
    866866      <FIELD name="name" arraysize="100" datatype="char">

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/batch.py

    r39094 r39224  
    104104
    105105        # create DVO tables (scratch database)
    106         self.scratchDb.createDvoTables()
     106        ### XXX self.scratchDb.createDvoTables() <-- remove this
    107107
    108108        # dump stuff to the log

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/cleanup.py

    r36697 r39224  
    5050            self.clean("FW")
    5151            self.clean("FO")
     52            self.clean("FG")
    5253
    5354            self.checkClientStatus()

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/diffbatch.py

    r39134 r39224  
    141141
    142142           # look up the path base once
    143            pathBase = gpc1Db.getDiffStagePathBase(skychunk.dvoLabel, int(diffSkyFileID))
     143           pathBase = gpc1Db.getDiffStagePathBase(int(diffSkyFileID))
    144144
    145145           # select the two (possible) cmf files (.cmf and .inv.cmf)
    … …  
    336336        else:
    337337            fluxScaleStr = "-1"
    338             detIDoffset = self.posImageDetCount
     338            detIDoffset = self.posImageDetCount[num]
    339339
    340340        if isPosFile or self.haveNegFits[num]:
    … …  
    723723        self.haveSrcPos = {}
    724724        self.haveNegFits = {}
    725         self.posImageDetCount = 0
     725        self.posImageDetCount = {}
    726726
    727727        for num in self.number:
    … …  
    742742            ## if we have a negative image (inv.cmf), load that as well:
    743743            self.haveNegFits[num] = 0
     744            self.posImageDetCount[num] = 0
    744745            if not self.negFits[num] == None:
    745746                fileNameNeg = self.negFits[num].getPath()
    … …  
    759760
    760761                posTableName = "SkyChip_psf_" + str(num)
    761                 self.posImageDetCount = self.scratchDb.getRowCount(posTableName)
     762                # self.posImageDetCount[num] = self.scratchDb.getMaxIPP_IDET(posTableName)
     763                self.posImageDetCount[num] = self.scratchDb.getRowCount(posTableName)
    762764
    763765            ## use stilts to get a list of the tables from the header

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/dvo.py

    r38995 r39224  
    7272            #     response = raw_input("(y/n) ")
    7373            #     if response == "y":
    74             self.resetAllTables()
     74            self.resetAllTables() ## called if we need to change DVOs
    7575
    7676    '''
    … …  
    9595        self.correctDvo = True
    9696        if not self.loadSkyTable():
     97            self.correctDvo = False
     98            return
     99        if not self.loadPhotcodes():
     100            self.correctDvo = False
     101            return
     102        if not self.loadImages():
    97103            self.correctDvo = False
    98104            return
    … …  
    298304            return False       
    299305
    300         # update the two meta tables, Images and SkyTable
    301         self.loadSkyTable()
     306        # update the SkyTable
     307        self.loadSkyTable() ## in setSkyArea
    302308
    303309        # set bounding-box coords
    … …  
    508514    def sync(self):
    509515
    510         # go no further if we've already partly ingested a different DVO
    511         if not self.correctDvo: return False
    512 
    513         # check we have something to do
    514         if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
    515             self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
    516             return True
    517 
    518         # make sure we have an up-to-date Images table
    519         self.loadImages()
    520 
    521         # purge stuff
    522         self.purgeRegions(self.regionsToPurge)
    523 
    524         totalRegions = len(self.regionsToIngest)
    525         if totalRegions > 0: self.logger.infoSeparator()
    526 
    527         # now loop through all regions and ingest to MySQL
    528         ingestedRegionsCount = 0
    529         regionCount = 1
    530         for region in self.regionsToIngest:
    531 
    532            self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
    533            if self.ingestRegion(region): ingestedRegionsCount += 1
    534            regionCount += 1
    535 
    536         self.logger.infoSeparator()
    537         self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
    538 
    539         # update lists after attempted sync
    540         self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
    541         self.printSummary()
     516        print "---- sync is disabled ----"
     517        os._exit(3)
     518
     519        ### XXX # go no further if we've already partly ingested a different DVO
     520        ### XXX if not self.correctDvo: return False
     521        ### XXX
     522        ### XXX # check we have something to do
     523        ### XXX if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1:
     524        ### XXX     self.logger.infoPair("Nothing to ingest or purge", "finished ingest")
     525        ### XXX     return True
     526        ### XXX
     527        ### XXX # make sure we have an up-to-date Images table
     528        ### XXX self.loadImages()
     529        ### XXX
     530        ### XXX # purge stuff
     531        ### XXX self.purgeRegions(self.regionsToPurge)
     532        ### XXX
     533        ### XXX totalRegions = len(self.regionsToIngest)
     534        ### XXX if totalRegions > 0: self.logger.infoSeparator()
     535        ### XXX
     536        ### XXX # now loop through all regions and ingest to MySQL
     537        ### XXX ingestedRegionsCount = 0
     538        ### XXX regionCount = 1
     539        ### XXX for region in self.regionsToIngest:
     540        ### XXX
     541        ### XXX    self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region))
     542        ### XXX    if self.ingestRegion(region): ingestedRegionsCount += 1
     543        ### XXX    regionCount += 1
     544        ### XXX
     545        ### XXX self.logger.infoSeparator()
     546        ### XXX self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest)))
     547        ### XXX
     548        ### XXX # update lists after attempted sync
     549        ### XXX self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec)
     550        ### XXX self.printSummary()
    542551
    543552        return True
    … …  
    667676    includes purging detections outside sky area
    668677    '''
    669     def nativeIngestDetections(self, boxId, boxDim):
     678    def nativeIngestDetections(self, boxId, boxDim, myBatchType):
    670679
    671680        # clear the 'ingested' field for all boxes owned by this host
    … …  
    688697
    689698        # make sure we have an up-to-date Images table
    690         self.loadImages()
     699        self.loadImages() # this is in nativeIngestDetections (remove?)
    691700
    692701        # the box dimensions are the area used to select the items of
    … …  
    695704        # to grab all detections / objects which are contributed by an
    696705        # item with the center in the given box
    697         dR_border = self.skychunk.BORDER / math.cos(math.radians(boxDim['DEC']))
    698         dD_border = self.skychunk.BORDER
    699        
    700         #This is a terrible hack but I'm tired of it taking a decade to ingest the FW skycells.
    701 
    702         #if self.batchType == "ST":
    703         #    dR_border = .4 / math.cos(math.radians(boxDim['DEC']))
    704         #    dD_border = .4
    705         #if self.batchType == "FW":
    706         #    dR_border = .4 / math.cos(math.radians(boxDim['DEC']))
    707         #    dD_border = .4
    708 
    709 
     706
     707        # P2 and DF need to use a full GPC1 exposure radius:
     708        myBorder = self.skychunk.BORDER_GPC1
     709
     710        # ST and FW need to use just a skycell
     711        if myBatchType == "ST" :
     712            myBorder = self.skychunk.BORDER_SKYCELL
     713        if myBatchType == "FW" :
     714            myBorder = self.skychunk.BORDER_SKYCELL
     715
     716        dR_border = myBorder / math.cos(math.radians(boxDim['DEC']))
     717        dD_border = myBorder
    710718
    711719        minRA  = boxDim['minRA']  - dR_border
    … …  
    788796            else:
    789797                cmd += " -photcode-start 12000 -photcode-end 12500"
    790         ## if (useFW == 1):
    791         ##     if (self.config.camera == "simtest"):
    792         ##         print "need photcodes for non gpc1 camera"
    793         ##     else:
    794         ##         print "need photcode for warp for gpc1 camera, default is no selection, should work, could be slow"
     798
     799        if (useST + useP2 + useDF + useFW > 1):
     800            self.logger.errorPair("multiple batch types in a run", "NOW ILLEGAL")
     801            os._exit(5)
    795802               
    796803        ## dvopsps can take a very long time. try 2 or 3 times to re-establish communication with the db before proceeding?

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/dvoobjects.py

    r38883 r39224  
    2828        # declare DVO file types of interest
    2929        self.ingestFileTypes = ['cpt', 'cps']
    30         self.loadPhotcodes()
     30        self.loadPhotcodes() # need to load photcodes when generating OB and other object-like batches
    3131
    3232    '''

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/forcedgalaxybatch.py

    r39127 r39224  
    7272
    7373    '''
     74    Generate ippObjID and make it an index
     75    '''
     76    def addIppObjID (self, tableName):
     77        sql = "ALTER TABLE " + tableName + " ADD COLUMN ippObjID BIGINT"
     78        self.scratchDb.execute(sql)
     79       
     80        sql = "CREATE INDEX ippObjID_index ON " + tableName + " ( ippObjID ) ";
     81        self.scratchDb.execute(sql)
     82
     83        sql = "UPDATE " + tableName + " set ippObjID = (OBJ_ID + (CAT_ID << 32))"
     84        self.scratchDb.execute(sql)
     85
     86    '''
    7487    Overriden from batch base class to import directly from DVO for forced galaxy shape parameters
    7588    '''
    … …  
    101114            self.logger.infoPair("0 rows for ","cpq table")
    102115            return False
     116
     117        # add ippObjID columns and index them:
     118        self.addIppObjID (cptTableName)
     119        self.addIppObjID (cpqTableName)
     120
    103121        return True
    104122
    … …  
    178196            # sqlLine.group("a." + filter[1] + "GalDecErr",       "b.DEC_ERR")
    179197           
    180             sql = sqlLine.makeEquals("WHERE a.ippObjID = (b.OBJ_ID + (b.CAT_ID << 32)) AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode))
     198            sql = sqlLine.makeEquals("WHERE a.ippObjID = b.ippObjID AND a.galModelType = b.MODEL_TYPE AND b.Photcode = " + str(photcode))
    181199           
    182200            self.logger.debugPair("cpq sql: ", sql)
    183201           
    184202            self.scratchDb.execute(sql)
     203
     204            ## # check on the status as we go:
     205            ## sql = "select count(*) as N, galModelType, nFilter, count(*)*nFilter as S from ForcedGalaxyShape group by galModelType, nFilter"
     206            ## rs = self.scratchDb.executeQuery(sql) 
     207            ##
     208            ## while rs.next():
     209            ##     N = rs.getInt(1)
     210            ##     M = rs.getInt(2)
     211            ##     F = rs.getInt(3)
     212            ##     S = rs.getInt(4)
     213            ##     
     214            ##     print "N: %d, TYPE: %d, nFilter: %d, Sum: %d" % (N, M, F, S)
    185215
    186216    '''
    … …  
    207237
    208238            sqlLine.group("objID",             "EXT_ID")
    209             sqlLine.group("ippObjID",          "OBJ_ID + (CAT_ID << 32)")
     239            sqlLine.group("ippObjID",          "ippObjID")
    210240            sqlLine.group("surveyID",          str(self.surveyID))
    211241            sqlLine.group("randomForcedGalID", "RAND("+str(self.batchID)+")")
    … …  
    215245            sql = sqlLine.makeRaw(") SELECT ", " FROM " + cptTableName)
    216246
    217             print "--- sql: " + sql
     247            # print "--- sql: " + sql
    218248
    219249            self.scratchDb.execute(sql)

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/gpc1db.py

    r39049 r39224  
    7979                   AND addRun.stage = '" + stage + "' \
    8080                   AND addRun.state = 'full' \
    81                    AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \
    82                    AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ") \
     81                   AND decl >= RADIANS(" + str(minDec) + ") AND decl < RADIANS(" + str(maxDec) + ") \
     82                   AND ra >= RADIANS(" + str(minRA) + ") AND ra < RADIANS(" + str(maxRA) + ") \
    8383                   AND dateobs >= '" + tstart + "'\
    8484                   AND dateobs <= '" + tend  + "'"
    … …  
    103103                   AND addRun.stage = '" + stage + "' \
    104104                   AND addRun.state = 'full' \
    105                    AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
    106                    AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     105                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
     106                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
    107107                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
    108108                   AND mjd_obs <= (to_days('" + tend + "') - 678941) "
    … …  
    121121                   JOIN mergedvodbProcessed using (merge_id) \
    122122                   JOIN diffInputSkyfile \
    123                    ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
    124                    AND stage = 'diff') \
     123                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \
    125124                   JOIN skycell using (skycell_id, tess_id) \
    126125                   JOIN warpRun on (warp1 = warp_id \
    … …  
    137136                   AND addRun.stage = '" + stage + "' \
    138137                   AND addRun.state = 'full' \
    139                    AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
    140                    AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     138                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
     139                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
    141140                   AND dateobs >= '" + tstart + "'\
    142141                   AND dateobs <= '" + tend  + "'\
    … …  
    164163                   AND addRun.stage = '" + stage + "' \
    165164                   AND addRun.state = 'full' \
    166                    AND decdeg BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \
    167                    AND radeg BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     165                   AND decdeg >= " + str(minDec) + " AND decdeg < " + str(maxDec) + " \
     166                   AND radeg >= " + str(minRA) + " AND radeg < " + str(maxRA) + " \
    168167                   AND mjd_obs >= (to_days('" + tstart + "')-678941) \
    169168                   AND mjd_obs <= (to_days('" + tend + "') - 678941) \
    … …  
    308307                   JOIN mergedvodbProcessed using (merge_id) \
    309308                   JOIN diffInputSkyfile \
    310                    ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 \
    311                    AND stage = 'diff') \
     309                   ON (diff_id = stage_id AND diff_skyfile_id = stage_extra1 AND stage = 'diff') \
    312310                   JOIN skycell using (skycell_id, tess_id) \
    313311                   JOIN warpRun on (warp1 = warp_id \
    … …  
    513511    get diff stage cmf name
    514512    '''
    515     def getDiffStagePathBase(self, dvoDb, diffSkyfileID):
     513    def getDiffStagePathBase(self, diffSkyfileID):
     514        sql = "SELECT DISTINCT diffSkyfile.path_base \
     515               FROM diffInputSkyfile \
     516               JOIN diffSkyfile using (diff_id, skycell_id) \
     517               WHERE diff_skyfile_id = " + str(diffSkyfileID)
     518
     519        rs = self.executeQuery(sql)
     520
     521        if not rs.first():
     522            self.logger.errorPair("missing cmf entry diff_skyfile_id:", str(diff_skyfile_id))
     523            # self.logger.errorPair("no diff cmfs found for diff_skyfile_id = %d" %diff_skyfile_id)
     524            return None
     525
     526        pathBase = rs.getString(1)
     527        return pathBase
     528
     529    '''
     530    get diff stage cmf name
     531    XXX EAM 20151119 : this function does too much work: you have a diffSkyfileID, that matches to a unique path_base
     532    '''
     533    def getDiffStagePathBaseOldVersion(self, dvoDb, diffSkyfileID):
    516534        print "got here, diffstagecmf"
    517535        sql = "SELECT DISTINCT diffSkyfile.path_base \
    518536               FROM mergedvodbRun join minidvodbRun using (minidvodb_id) \
    519537               JOIN addRun using (minidvodb_name) \
    520                JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \
    521                AND diff_id = stage_id AND stage = 'diff') \
    522                JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \
     538               JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 AND diff_id = stage_id AND stage = 'diff') \
     539               JOIN diffSkyfile using (diff_id, skycell_id) \
    523540               WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \
    524541               AND minidvodbRun.state = 'merged' \
    … …  
    561578    '''
    562579    get diff stage cmf name
     580    XXX EAM 20151119 This function is broken AND it does too much work
    563581    '''
    564582    def getDiffStageCmf(self, dvoDb, diffSkyfileID):

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/ipptopspsdb.py

    r39008 r39224  
    324324               AND processed = 1 \
    325325               AND loaded_to_datastore = 1 \
    326                AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()"
     326               AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()"
    327327
    328328        total = -1
    … …  
    421421               AND timestamp > '" + self.skychunk.epoch + "' \
    422422               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    423                AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()"
     423               AND timestamp >= now() - INTERVAL " + interval + " AND timestamp < now()"
    424424
    425425        try:
    … …  
    545545               AND dvo_db = '" + self.skychunk.dvoLabel + "' \
    546546               AND batch_type != 'IN' \
    547                AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()"
     547               AND timestamp >= now() - INTERVAL 12 HOUR AND timestamp < now()"
    548548
    549549        try:
    … …  
    11581158
    11591159        self.skychunk.boxSize = rs.getDouble(10)
    1160         # self.skychunk.halfBox = self.skychunk.boxSize/2.0
    1161         # self.skychunk.boxSizeWithBorder = self.skychunk.boxSize + (self.skychunk.BORDER * 2)
    11621160
    11631161        self.skychunk.basePath = rs.getString(11)
    … …  
    12401238        sql = "SELECT id FROM box \
    12411239               WHERE skychunk  = '" + self.skychunk.name + "' \
    1242                AND ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
    1243                AND dec_center BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
     1240               AND ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
     1241               AND dec_center >= " + str(minDEC) + " AND dec_center < " + str(maxDEC)
    12441242
    12451243               # AND box_side = " + str(self.skychunk.boxSize)
    … …  
    14241422               FROM box \
    14251423               JOIN pending ON (id = box_id) \
    1426                WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     1424               WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
    14271425               AND batch_type = '" + batchType + "' \
    14281426               AND skychunk = '" + self.skychunk.name + "' \
    … …  
    15031501               FROM box \
    15041502               JOIN pending ON (id = box_id) \
    1505                WHERE ra_center BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
     1503               WHERE ra_center >= " + str(minRA) + " AND ra_center < " + str(maxRA) + " \
    15061504               AND skychunk = '" + self.skychunk.name + "' \
    15071505               ORDER BY dec_center DESC"
    … …  
    17121710        sql = "SELECT DISTINCT stage_id \
    17131711               FROM all_pending \
    1714                WHERE ra_bore BETWEEN " + str(minRA) + " AND " + str(maxRA) + " \
    1715                AND dec_bore BETWEEN " + str(minDEC) + " AND " + str(maxDEC)
     1712               WHERE ra_bore >= " + str(minRA) + " AND ra_bore < " + str(maxRA) + " \
     1713               AND dec_bore >= " + str(minDEC) + " AND dec_bore < " + str(maxDEC)
    17161714
    17171715        try:

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/loader.py

    r39008 r39224  
    172172                    if (batchType == "P2" or batchType == "ST" or batchType =="DF" or batchType == "FW"):
    173173                        if self.config.resetDvo or not self.ippToPspsDb.isBoxIngested(boxId, self.scratchDb.dbHost):
    174                             if not self.dvo.nativeIngestDetections(boxId, boxDim):
     174                            if not self.dvo.nativeIngestDetections(boxId, boxDim, batchType):
    175175                                self.logger.error("Unable to ingest detections with nativeIngest")
    176176                                return False

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/queue.py

    r38987 r39224  
    146146
    147147                    # XXX not sure why this is not done with a for-loop...
     148                    # answer: because python does not have a good float-style for-loop
    148149                    while dec <= self.skychunk.maxDec:
    149150                        while ra <= self.skychunk.maxRa:
    … …  
    158159           
    159160                           ids = self.ippToPspsDb.getItemsInThisBox(minRA, maxRA, minDEC, maxDEC)
    160    
     161
    161162                           self.logger.info("|    %5.1f    |    %5.1f    |  %9d  |  %9d  |" % (
    162163                                       ra,
    … …  
    165166                                       len(ids)))
    166167               
    167                            if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids)
    168                            # print "inserted pending"
     168                           if len(ids) > 0:
     169                               ## for myID in ids:
     170                               ##     print "id: %d : %10.6f - %10.6f | %10.6f - %10.6f" % (myID, minRA, maxRA, minDEC, maxDEC)
     171
     172                               self.ippToPspsDb.insertPending(box_id, batchType, ids)
     173                               # print "inserted pending"
     174
    169175                           ra = ra + 2*dR
    170176                           # print "new ra: ", ra

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/scratchdb.py

    r39110 r39224  
    406406
    407407        return size
     408
     409    '''
     410    Returns a row count for this table
     411    '''
     412    def getMaxIPP_IDET(self, table):
     413
     414        sql = "SELECT MAX(IPP_IDET) FROM " + table
     415        rs = self.executeQuery(sql) 
     416
     417        if not rs.first():
     418            self.logger.exception("Could not count rows for table: '" + table + "'")
     419            return 0
     420
     421        return rs.getInt(1)
    408422
    409423    '''
    … …  
    655669    def resetAllDvoTables(self):
    656670
    657        # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
    658        self.logger.infoPair("Dropping table", self.dvoImagesTable)
    659        self.dropTable(self.dvoImagesTable)
    660        self.logger.infoPair("Dropping table", self.dvoSkyTable)
    661        self.dropTable(self.dvoSkyTable)
    662        self.logger.infoPair("Dropping table", self.dvoDoneTable)
    663        self.dropTable(self.dvoDoneTable)
    664 
    665        # blow away existing dvoDetection table & re-crate
    666        if not self.resetDvoDetectionTable():
    667            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
    668            return False
    669 
    670        # create dvoDone table
    671        self.logger.infoPair("Creating table", self.dvoDoneTable)
    672        sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
    673        try: self.execute(sql)
    674        except:
    675            self.logger.errorPair("Unable to create table", self.dvoDoneTable)
    676            return False
    677 
    678        self.changeEngineToInnoDB(self.dvoDoneTable)
    679 
    680        return True
     671        # re-create the definition tables
     672        self.createDvoTables()
     673
     674        # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint
     675        self.logger.infoPair("Dropping table", self.dvoImagesTable)
     676        self.dropTable(self.dvoImagesTable)
     677        self.logger.infoPair("Dropping table", self.dvoSkyTable)
     678        self.dropTable(self.dvoSkyTable)
     679        self.logger.infoPair("Dropping table", self.dvoDoneTable)
     680        self.dropTable(self.dvoDoneTable)
     681
     682        # blow away existing dvoDetection table & re-crate
     683        if not self.resetDvoDetectionTable():
     684            self.logger.errorPair("Unable to reset scratch table", "DvoDetectionFull")
     685            return False
     686
     687        # create dvoDone table
     688        self.logger.infoPair("Creating table", self.dvoDoneTable)
     689        sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)"
     690        try:
     691            self.execute(sql)
     692        except:
     693            self.logger.errorPair("Unable to create table", self.dvoDoneTable)
     694            return False
     695
     696        self.changeEngineToInnoDB(self.dvoDoneTable)
     697           
     698        return True
    681699
    682700    '''

  • branches/eam_branches/ipp-20151113/ippToPsps/jython/skychunk.py

    r38763 r39224  
    3939        # XXX this is a poor place to put this information -- it is completely gpc-specific. 
    4040        # probably should go elsewhere.  This number is scaled in loader.py by cos(dec)
    41         self.BORDER = 1.65
     41        self.BORDER_GPC1    = 1.65
     42        self.BORDER_SKYCELL = 0.40
    4243        self.isLoaded = False
    4344

  • branches/eam_branches/ipp-20151113/ippToPsps/test

    • Property svn:ignore

      •  

        old new  
        88stackbatch.dat
        99objectbatch.dat
         10diffbatch.dat
         11diffobjbatch.dat
         12forcebatch.dat
         13forcedobjectbatch.dat

  • branches/eam_branches/ipp-20151113/ippToPsps/test/fulltest.sh

    r39061 r39224  
    561561
    562562  rm -rf $OUTDIR/FO
     563  rm -rf $OUTDIR/FG
    563564
    564565  ippjython loader.py $options $forcedobjqueuename once -simtest < /dev/null

  • branches/eam_branches/ipp-20151113/ippToPsps/test/gpc1schema.sql

    r39015 r39224  
    2121  CREATE TABLE diffRun             ( diff_id INT, tess_id CHAR(64), state CHAR(16), diff_mode INT, software_ver CHAR(16) );
    2222  CREATE TABLE diffInputSkyfile    ( diff_id INT, warp1 INT, stack2 INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64) );
    23   CREATE TABLE diffSkyfile         ( diff_id INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64), path_base CHAR(255) );
     23  CREATE TABLE diffSkyfile         ( diff_id INT, skycell_id CHAR(64), path_base CHAR(255) );
    2424
    2525  CREATE TABLE addRun              ( add_id INT, stage CHAR(64), stage_extra1 INT, state CHAR(64), stage_id INT, minidvodb_name CHAR(64) );

  • branches/eam_branches/ipp-20151113/ippToPsps/test/mkgpc1data.dvo

    r39105 r39224  
    654654  dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M")
    655655  dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID,  $SKYCELL )
    656   dbinsert diffSkyfile (diff_id, diff_skyfile_id, tess_id, skycell_id, path_base) values ($difID, $diffSkyfileID, $TESS_ID, $SKYCELL, "$filename")
     656  dbinsert diffSkyfile (diff_id, skycell_id, path_base) values ($difID, $SKYCELL, "$filename")
    657657end
    658658
Note: See TracChangeset for help on using the changeset viewer.