IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39922


Ignore:
Timestamp:
Jan 6, 2017, 11:16:30 AM (10 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/czw_branch/20160809/ippToPsps
Files:
19 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20160809/ippToPsps/config/Makefile.am

    r39010 r39922  
    1010  tables.FO.vot \
    1111  tables.FG.vot \
     12  tables.GO.vot \
    1213  settings.xml
    1314

  • branches/czw_branch/20160809/ippToPsps/config/changes.txt

    r36201 r39922  
    99
    1010-- not yet alter table batch add column smf_version varchar(16) after comment;
     11
     12alter table skychunk add column queue_FG tinyint after queue_FO;
     13alter table skychunk add column queue_GO tinyint after queue_FG;
     14
     15alter table skychunk change column queue_FG queue_FG tinyint(1);
     16alter table skychunk change column queue_GO queue_GO tinyint(1);
     17alter table skychunk change column queue_GO queue_GO tinyint(1) default 0;
     18alter table skychunk change column queue_FO queue_FO tinyint(1) default 0;
     19alter table skychunk change column queue_FG queue_FG tinyint(1) default 0;

  • branches/czw_branch/20160809/ippToPsps/config/ippToPspsDbSchema.sql

    r38823 r39922  
    111111  `queue_FW` tinyint(1) default '0',
    112112  `queue_FO` tinyint(1) default '0',
     113  `queue_FG` tinyint(1) default '0',   
     114  `queue_GO` tinyint(1) default '0',
    113115  `active` tinyint(1) default '1',
    114116  `parallel` tinyint(1) default '0',
    … …  
    217219) ENGINE=InnoDB DEFAULT CHARSET=latin1;
    218220SET character_set_client = @saved_cs_client;
     221--
     222-- Table structure for table `forcedgalobject`
     223--
     224DROP TABLE IF EXISTS `forcedgalobject`;
     225SET @saved_cs_client     = @@character_set_client;
     226SET character_set_client = utf8;
     227CREATE TABLE `forcedgalobject` (
     228  `batch_id` bigint(20) unsigned NOT NULL,
     229  `region` varchar(50) default NULL,
     230  KEY `batch_id` (`batch_id`),
     231  CONSTRAINT `forcedgalobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
     232) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     233SET character_set_client = @saved_cs_client;
     234
     235
     236
     237--
     238-- Table structure for table `gaiaobject`
     239--
     240DROP TABLE IF EXISTS `gaiaobject`;
     241SET @saved_cs_client     = @@character_set_client;
     242SET character_set_client = utf8;
     243CREATE TABLE `gaiaobject` (
     244  `batch_id` bigint(20) unsigned NOT NULL,
     245  `region` varchar(50) default NULL,
     246  KEY `batch_id` (`batch_id`),
     247  CONSTRAINT `gaiaobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`)
     248) ENGINE=InnoDB DEFAULT CHARSET=latin1;
     249SET character_set_client = @saved_cs_client;
     250
    219251
    220252--

  • branches/czw_branch/20160809/ippToPsps/config/schema_browser.vot

    r39210 r39922  
    171171  <RESOURCE name="Derived Data Products">
    172172     <DESCRIPTION>These tables contain values that are derived from the fundamantal data products.</DESCRIPTION>
     173     <TABLE name="GaiaFrameCoordinate"> </TABLE>
     174
    173175     <TABLE name="pcsAnnZ"> </TABLE>
    174176     <TABLE name="pcsPhotoZ"> </TABLE>

  • branches/czw_branch/20160809/ippToPsps/config/settings.xml

    r39644 r39922  
    819819  <!-- ippToPsps Db section (REAL version) -->
    820820  <ipptopspsdatabase>
    821     <name>ipptopsps2</name>
     821    <name>ipptopsps3</name>
    822822    <host>ipp006</host>
    823823    <user>dvo</user>

  • branches/czw_branch/20160809/ippToPsps/jython/Makefile.am

    r39007 r39922  
    1919  dvoforcedgalaxy.py \
    2020  dvoobjects.py \
     21  dvogaiaobjects.py \
    2122  dvo.py \
    2223  dxlayer.py \
    … …  
    2526  forcedwarpbatch.py \
    2627  forcedgalaxybatch.py \
     28  gaiaobjectbatch.py \
    2729  gpc1db.py \
    2830  initbatch.py \

  • branches/czw_branch/20160809/ippToPsps/jython/batch.py

    r39315 r39922  
    700700        # for OB, this imports object data from DVO
    701701        if not self.importIppTables():
    702             if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG"):
     702            if (self.batchType == "OB" or self.batchType =="DO" or self.batchType =="FO" or self.batchType =="FG" or self.batchType =="GO"):
    703703                self.logger.errorPair("skipping this batch", "no dvo tables for this region")
    704704                self.ippToPspsDb.updateProcessed(self.batchID, 1)

  • branches/czw_branch/20160809/ippToPsps/jython/cleanup.py

    r39154 r39922  
    5151            self.clean("FO")
    5252            self.clean("FG")
     53            self.clean("GO")
    5354
    5455            self.checkClientStatus()

  • branches/czw_branch/20160809/ippToPsps/jython/dvo.py

    r39467 r39922  
    152152        self.scratchDb.execute("ALTER TABLE " + self.scratchDb.dvoImagesTable + " ADD PRIMARY KEY (IMAGE_ID, EXTERN_ID)")
    153153
     154        self.scratchDb.execute("CREATE INDEX externID_index ON dvoImagesFull (EXTERN_ID)")
    154155        self.scratchDb.setImportedThisDvoTable(path)
    155156        return True

  • branches/czw_branch/20160809/ippToPsps/jython/fits.py

    r39719 r39922  
    147147
    148148        if False:
     149            self.logger.infoPair("running findAndReadHeaderRaw",name)
    149150            status = self.findAndReadHeaderRaw(name, VERBOSE)
    150151            return status
    151 
     152        self.logger.infoPair("running findAndReadHeaderOhana",name)
    152153        status = self.findAndReadHeaderOhana(name, VERBOSE)
    153154        return status

  • branches/czw_branch/20160809/ippToPsps/jython/gpc1db.py

    r39544 r39922  
    659659
    660660
     661    '''
     662    Gets the exp_id from teh exp_name
     663    '''
     664    def getExpIdFromExpName(self, expName):
     665       sql = "SELECT exp_id from rawExp where exp_name = '"+expName+"'"
     666       try:
     667           rs = self.executeQuery(sql)
     668       except:
     669           self.logger.infoPair("failed sql:",sql)
     670           self.logger.exception("Can't query for imageIDs")
     671           
     672
     673       expIDs = []
     674       while (rs.next()):
     675           expIDs.append(rs.getString(1))
     676       rs.close()
     677       expID = expIDs[0]
     678       return expID
     679
     680   
     681
     682
    661683
    662684    '''

  • branches/czw_branch/20160809/ippToPsps/jython/insert.dvodb.info.py

    r38884 r39922  
    8383                # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo
    8484            #for object-like batches, info comes from dvo
    85                 if (batchType == "OB" or batchType =="DO" or batchType =="FO"):
     85                if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"):
    8686                    self.dvoObjects.setSkyArea(
    8787                            self.skychunk.minRa,

  • branches/czw_branch/20160809/ippToPsps/jython/ipptopspsdb.py

    r39158 r39922  
    819819
    820820        sql = "INSERT INTO diffobject ( \
     821               batch_id \
     822               ,region \
     823               ) VALUES ( \
     824               " + str(batchID) + " \
     825               ,'" + region + "' \
     826               )"
     827
     828        self.execute(sql)
     829
     830    '''
     831    Inserts some gaiaobject metadata for this batch ID
     832    '''
     833    def insertGaiaObjectMeta(self, batchID, region):
     834
     835        sql = "INSERT INTO gaiaobject ( \
    821836               batch_id \
    822837               ,region \
    … …  
    11041119        ,queue_FW \
    11051120        ,queue_FO \
     1121        ,queue_FG \
     1122        ,queue_GO \
    11061123        ,parallel \
    11071124        ,P2_smf_version \
    … …  
    11771194        if rs.getInt(23) == 1: self.skychunk.batchTypes.append("DO")
    11781195        if rs.getInt(24) == 1: self.skychunk.batchTypes.append("FW")
    1179         if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FG")
     1196###        if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FG") fix this when P2 is done
    11801197        if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO")
     1198        if rs.getInt(26) == 1: self.skychunk.batchTypes.append("FG")
     1199        if rs.getInt(27) == 1: self.skychunk.batchTypes.append("GO")
     1200       
    11811201
    11821202        self.skychunk.force = True # TODO
    11831203        self.skychunk.parallel = False # TODO
    11841204
    1185         if rs.getInt(26) == 1: self.skychunk.parallel = True
    1186         self.skychunk.P2_smf_version = rs.getString(27)
     1205        if rs.getInt(28) == 1: self.skychunk.parallel = True
     1206        self.skychunk.P2_smf_version = rs.getString(29)
    11871207        # options: "use_new", "not_reproc", "use_original"
    11881208
    1189         self.skychunk.ST_cmf_version = rs.getString(28)
     1209        self.skychunk.ST_cmf_version = rs.getString(30)
    11901210        # options: not used?
    11911211
    … …  
    12121232        # print "typename: " + rsmd.getColumnTypeName(16)
    12131233
    1214         self.skychunk.trange_start = rs.getString(29)
     1234        self.skychunk.trange_start = rs.getString(31)
    12151235        # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00"
    12161236
    1217         self.skychunk.trange_end = rs.getString(30)
     1237        self.skychunk.trange_end = rs.getString(32)
    12181238        # options: "YYYY-MM-DD,HH:MM:SS" or "0000-00-00 00:00:00"
    12191239

  • branches/czw_branch/20160809/ippToPsps/jython/loader.py

    r39507 r39922  
    3030from forcedobjectbatch import ForcedObjectBatch
    3131from forcedgalaxybatch import ForcedGalaxyBatch
     32from gaiaobjectbatch import GaiaObjectBatch
     33
    3234#
    3335#
    … …  
    302304                                              id,
    303305                                              batchID)
     306                elif batchType == "GO":
     307                    batch = GaiaObjectBatch(self.logger,
     308                                              self.config,
     309                                              self.skychunk,
     310                                              self.gpc1Db,
     311                                              self.ippToPspsDb,
     312                                              self.scratchDb,
     313                                              id,
     314                                              batchID)
    304315                   
    305316                batch.run()

  • branches/czw_branch/20160809/ippToPsps/jython/metrics.py

    r38969 r39922  
    5353            rateFO = self.printStats("FO")
    5454            rateFG = self.printStats("FG")
     55            rateGO = self.printStats("GO")
    5556            self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+")
    5657       
    … …  
    7677            pendFOProcessed = self.printTableRow(stages, "FO")
    7778            pendFGProcessed = self.printTableRow(stages, "FG")
     79            pendGOProcessed = self.printTableRow(stages, "GO")
    7880            self.writeTableSeparator(colCount)
    7981
    … …  
    9395            try: self.logger.infoPair("Estimated time for FOs", "%.1f hours" % (pendFOProcessed / rateFO))
    9496            except: pass
    95             try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG))
     97            try: self.logger.infoPair("Estimated time for FGs", "%.1f hours" % (pendFGProcessed / rateFG))
     98            try: self.logger.infoPair("Estimated time for GOs", "%.1f hours" % (pendGOProcessed / rateGO))
    9699            except: pass
    97100       
    … …  
    107110            self.plot.createDensityPlot("FO", True)
    108111            self.plot.createDensityPlot("FG", True)
     112            self.plot.createDensityPlot("GO", True)
     113           
    109114            # some clean-up before going to sleep
    110115            self.ippToPspsDb.purgeDeadClients()

  • branches/czw_branch/20160809/ippToPsps/jython/plotter.py

    r36697 r39922  
    3232            self.plot.createDensityPlot("FW")
    3333            self.plot.createDensityPlot("FO")
     34            self.plot.createDensityPlot("FG")
     35            self.plot.createDensityPlot("GO")
     36           
    3437
    3538    '''

  • branches/czw_branch/20160809/ippToPsps/jython/pollOdm.py

    r38969 r39922  
    4242        else: self.stages = [self.STAGE]
    4343
    44         if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG']
     44        if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO', 'FG', 'GO']
    4545        else: self.batchTypes = [self.BATCHTYPE]
    4646
    … …  
    126126    '''
    127127    def printUsage(self):
    128         super(PollOdm, self).printUsage("<IN|P2|ST|OB|DF|DO|FW|FO|FG|all> <unloaded|unmergeworthy|unmerged|all> [hours]")
     128        super(PollOdm, self).printUsage("<IN|P2|ST|OB|DF|DO|FW|FO|FG|GO|all> <unloaded|unmergeworthy|unmerged|all> [hours]")
    129129   
    130130   

  • branches/czw_branch/20160809/ippToPsps/jython/queue.py

    r39159 r39922  
    7373
    7474                # for object-like batches, info comes from dvo
    75                 if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG"):
     75                if (batchType == "OB" or batchType =="DO" or batchType =="FO" or batchType == "FG" or batchType == "GO"):
    7676                    self.dvoObjects.setSkyArea(
    7777                            self.skychunk.minRa,

  • branches/czw_branch/20160809/ippToPsps/jython/stackbatch.py

    r39649 r39922  
    129129           self.zpImage[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_OBS')
    130130           self.zpError[filter] = self.getKeyFloat(header, "%.8f", 'ZPT_ERR')
    131 
    132            deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test)
    133            if not deteffHeader:
    134                if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
    135                return False
    136 
    137            self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
    138            self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
     131           hasdeteff = 1           
     132           try: deteffHeader = self.fits[filter].findAndReadHeader("SkyChip.deteff", self.config.test)
     133           except:
     134               hasdeteff = 0
     135           #if not deteffHeader:
     136           #    if self.config.test: self.logger.errorPair("No header found for Skychip.deteff for", filter)
     137           #    return False
     138               
     139           if hasdeteff == 1:         
     140               self.magref[filter] = self.getKeyFloat(deteffHeader, "%.8f", 'DETEFF.MAGREF')
     141               self.nInjected[filter] = int(self.getKeyInt(deteffHeader, 0, 'DETEFF.NUM'))
    139142
    140143           ## MEH -- md5sum test
    … …  
    733736        # sersic fit has an extra parameter
    734737        if (ippModelType == "PS_MODEL_SERSIC"):
    735             sqlLine.group("a." + filter + model + "Nu",    "1 / (2.0 * b.EXT_PAR_07)")
    736             sqlLine.group("a." + filter + model + "NuErr", "'0.1'") # this is not measured at the moment
     738            testSQL = 'show columns FROM rSkyChip_xfit LIKE "EXT_PAR_07"'
     739            have_ext_par_07 = 1
     740            try:
     741                rs = self.scratchDb.execute(sql)
     742            except:
     743                have_ext_par_07 = 0
     744            if have_ext_par_07 == 1:
     745                sqlLine.group("a." + filter + model + "Nu",    "1 / (2.0 * b.EXT_PAR_07)")
     746                sqlLine.group("a." + filter + model + "NuErr", "'0.1'") # this is not measured at the moment
     747
     748            ### I think, if ippModelType = PS_MODEL_SERSIC, and EXT_PAR_07 does not exist, then Nu, Nerr = null
     749           
    737750
    738751        # missing these two fields?
    … …  
    11701183                # search the words for the o....g....o part
    11711184                for j in range(len(words)):
    1172                     match = re.search('o\d\d\d\dg\d\d\d\d\o', words[j])
    1173                     if (match != None):
     1185                    match1 = re.search('^\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading  o: then ones with borked no os and cs will work now
     1186                    match2 = re.search('\o\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading  o: then ones with borked no os and cs will work now
     1187                    match3 = re.search('\c\d\d\d\dg\d\d\d\d\o', words[j]) #remove leading  o: then ones with borked no os and cs will work now
     1188                    if (match1 != None):
    11741189                        ogo=j
     1190                        ogoname = 'o'+words[ogo]
     1191                        break
     1192                    if (match2 != None):
     1193                        ogo=j
     1194                        ogoname = words[ogo]
     1195                        break
     1196                    if (match3 != None):
     1197                        ogo=j
     1198                        ogoname = words[ogo]
    11751199                        break
    11761200                    # did we find it? if so, are there are least 2 more elements past the ogo?
    … …  
    11781202
    11791203                if ogo > -1:
    1180                     if (ogo +1) < len(words):
     1204                    if (ogo +2) < len(words): #this was the mistake before
    11811205                        # there is a o....g....o element, and the next 2 exist
    11821206                        frameID = words[ogo+1]
    11831207                       
    1184                     else:
    1185                         self.logger.infoPair("unable to find exp_id from", inputName)
    1186                         raise
     1208                    else:
     1209                        #ogoname = "o"+words[ogo]
     1210                        try: frameID = self.gpc1Db.getExpIdFromExpName(ogoname)
     1211                        except:
     1212                            self.logger.error("cant get expid")
     1213                            raise
     1214                       
     1215#                        frameID = str(rs.getInt(1))
     1216       
     1217#                        self.logger.infoPair("unable to find exp_id from", inputName)
     1218#                        raise
    11871219                        # and then raise an error and stop the program. I expect this to mostly work fine
    11881220        # for the extra long label (?) - can probably get it from just the o g o (if exp_id
    … …  
    12161248
    12171249        self.tablesToExport.append("StackToFrame")
     1250        return True
    12181251
    12191252    '''
    … …  
    14961529            if self.stackIDs[filter] > 0:       
    14971530                self.populateStackMeta(filter)
    1498 
     1531               
    14991532        self.populateStackObjectThin()
    15001533
    … …  
    15291562        self.logger.infoPair("populating","StackToImage")
    15301563        self.populateStackToImage()
    1531        
     1564       
     1565 
     1566       
    15321567        self.logger.infoPair("populating","StackToFrame")
    1533         self.populateStackToFrame()
    1534        
     1568
     1569#        self.populateStackToFrame()
     1570        if not self.populateStackToFrame():
     1571                    self.logger.errorPair("can't populate"," stack to frame")
     1572                    return False
     1573        self.logger.infoPair("removing frameID"," duplicates")
     1574        self.scratchDb.execute("ALTER IGNORE TABLE StackToFrame ADD UNIQUE INDEX(frameID)")
     1575
    15351576        self.setMinMaxObjID(["StackObjectThin"])
    15361577
Note: See TracChangeset for help on using the changeset viewer.