IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38990


Ignore:
Timestamp:
Oct 28, 2015, 7:55:59 AM (11 years ago)
Author:
eugene
Message:

enable ohana fits_to_mysql

Location:
trunk/ippToPsps
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/batch.py

    r38987 r38990  
    452452    Accepts a regular expression filter so not all tables need to be imported
    453453    '''
     454    def importIppTables(self, columns="*", filter=""):
     455        if self.useOhanaMYSQLtoFITS:
     456            if not self.importIppTablesOhana():
     457                self.logger.errorPair("problem running", "importIppTablesOhana")
     458                return False
     459        else:
     460            if not self.importIppTablesStilts(columns, filter):
     461                self.logger.errorPair("problem running", "importIppTablesStilts")
     462                return False
     463
     464        return True
     465
     466    '''
     467    Imports IPP tables from FITS file
     468
     469    Accepts a regular expression filter so not all tables need to be imported
     470    '''
    454471    def importIppTablesOhana(self):
    455472       
    … …  
    463480            return False
    464481
    465         loadFITStoMYSQL(self.fits.getPath(), "none")
     482        self.scratchDb.loadFITStoMYSQL(self.fits.getPath(), "none")
    466483
    467484        ## may need to do this with direct sql:
    … …  
    496513    Accepts a regular expression filter so not all tables need to be imported
    497514    '''
    498     def importIppTables(self, columns="*", filter=""):
     515    def importIppTablesStilts(self, columns="*", filter=""):
    499516       
    500517        self.logger.infoPair("Importing tables with filter", filter)

  • trunk/ippToPsps/jython/config.py

    r38973 r38990  
    2121    def __init__(self):
    2222
     23        self.useOhanaMYSQLtoFITS = False
    2324        self.test = False
    2425        self.resetDvo = False

  • trunk/ippToPsps/jython/dvo.py

    r38981 r38990  
    589589    Imports the Photcode file
    590590    '''
     591    def importPhotcodes(self, path, tableName):
     592        if True:
     593            self.importPhotcodesOhana(path,tableName)
     594        else:
     595            self.importPhotcodesStilts(path,tableName)
     596
     597    '''
     598    Imports the Photcode file
     599    '''
    591600    def importPhotcodesOhana(self, path, tableName):
    592601
    593       self.logger.infoPair("Importing file", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path))))
     602      self.logger.infoPair("Importing file with ohana", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path))))
    594603      self.logger.debugPair("Writing to database table", tableName)
    595604
    … …  
    598607      # table = stilts.tpipe(table, cmd='addcol -after TYPE TYPE_AS_INT "TYPE + 0"')
    599608
    600       loadFITStoMYSQL (path, tableName)
     609      self.scratchDb.loadFITStoMYSQL (path, tableName)
    601610
    602611      self.logger.debugPair("Finished importing", tableName)
    … …  
    607616    Imports the Photcode file
    608617    '''
    609     def importPhotcodes(self, path, tableName):
     618    def importPhotcodesStilts(self, path, tableName):
    610619
    611620      self.logger.infoPair("Importing file", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path))))

  • trunk/ippToPsps/jython/forcedwarpbatch.py

    r38960 r38990  
    459459        results= self.scratchDb.reportAndDeleteRowsWithNULLS("ForcedWarpMeasurement_"+forcedWarpID, "objID")
    460460        rowCountAfter =  self.scratchDb.getRowCount("ForcedWarpMeasurement_"+forcedWarpID)
    461         self.logger.infoPair("row count of ForcedWarpMeasurement",rowCountBefore)
     461        self.logger.infoPair("row count of ForcedWarpMeasurement", rowCountBefore)
    462462        self.logger.infoPair("row count of ForcedWarpMeasurement after cull of null objID ", rowCountAfter)
    463463        if rowCountAfter == 0:
    464464            self.skipBatch = True
    465             self.logger.infoPair ("skip this batch, it is bad")
     465            self.logger.infoPair ("skip this batch", "it is bad")
    466466            return False
    467467#        self.generateRandomIDs()

  • trunk/ippToPsps/jython/scratchdb.py

    r38981 r38990  
    55import os
    66import logging
     7import tempfile
     8from subprocess import call, PIPE, Popen
    79
    810from java.lang import *
    … …  
    809811    def loadFITStoMYSQL(self, filename, tablename):
    810812
     813        self.logger.infoPair("starting loadFITStoMYSQL: ", filename)
     814
    811815        # TODO path to DVO prog hardcoded temporarily
    812         cmd = "fits_to_mysql"
    813         cmd += " -dbhost " + self.scratchDb.dbHost
    814         cmd += " -dbname " + self.scratchDb.dbName
    815         cmd += " -dbuser " + self.scratchDb.dbUser
    816         cmd += " -dbpass " + self.scratchDb.dbPass
    817         cmd += filename
     816        cmd = "fits_to_mysql " + filename
     817        cmd += " -dbhost " + self.dbHost
     818        cmd += " -dbname " + self.dbName
     819        cmd += " -dbuser " + self.dbUser
     820        cmd += " -dbpass " + self.dbPass
     821
     822        self.logger.debugPair("preparing command: ", cmd)
    818823
    819824        if tablename != "none":
    … …  
    822827        (stdoutFile, stdoutName) = tempfile.mkstemp(prefix="fits_to_mysql.stdout.", dir="/tmp")
    823828        (stderrFile, stderrName) = tempfile.mkstemp(prefix="fits_to_mysql.stderr.", dir="/tmp")
     829
     830        self.logger.debugPair("generated tmpfiles: ", cmd)
    824831
    825832        self.logger.infoPair("Running fits_to_mysql", cmd)
    … …  
    838845            os._exit(1)
    839846
    840         os.close(loadFITStoMYSQL)
    841         os.close(loadFITStoMYSQL)
     847        os.close(stdoutFile)
     848        os.close(stderrFile)

  • trunk/ippToPsps/test/fulltest.sh

    r38974 r38990  
    2929set forcedobjqueue = 1
    3030set forcedobjbatch = 1
     31
     32set debug = ""
    3133
    3234set targ = ($argv)
    … …  
    8991    set forcedobjqueue = 0
    9092    set forcedobjbatch = 0
     93    breaksw;
     94  case -debug
     95    set debug = "-debug"
    9196    breaksw;
    9297  default:
    … …  
    246251  echo "2000-01-01 00:00:00.0  " >> initbatch.dat # trange_start
    247252  echo "2020-01-01 00:00:00.0  " >> initbatch.dat # trange_end
    248   ippjython queue.py -test edit < initbatch.dat
     253  ippjython queue.py $debug -test edit < initbatch.dat
    249254  if ($status) exit 1
    250255
    … …  
    253258
    254259  rm -rf $OUTDIR/IN
    255   ippjython loader.py -test initbatch init -simtest < /dev/null
     260  ippjython loader.py $debug -test initbatch init -simtest < /dev/null
    256261  if ($status) exit 1
    257262endif
    … …  
    296301  echo "2000-01-01 00:00:00.0 " >> cambatch.dat # trange_start
    297302  echo "2020-01-01 00:00:00.0 " >> cambatch.dat # trange_end
    298   ippjython queue.py -test edit -simtest < cambatch.dat
     303  ippjython queue.py $debug -test edit -simtest < cambatch.dat
    299304  if ($status) exit 1
    300305endif
    … …  
    305310
    306311  rm -rf $OUTDIR/P2
    307   ippjython loader.py -test $queuename once -simtest < /dev/null
     312  ippjython loader.py $debug -test $queuename once -simtest < /dev/null
    308313  if ($status) exit
    309314endif
    … …  
    348353  echo "2000-01-01 00:00:00.0 " >> objectbatch.dat # trange_start
    349354  echo "2020-01-01 00:00:00.0 " >> objectbatch.dat # trange_end
    350   ippjython queue.py -test edit once -simtest < objectbatch.dat
     355  ippjython queue.py $debug -test edit once -simtest < objectbatch.dat
    351356  if ($status) exit
    352357endif
    … …  
    358363  rm -rf $OUTDIR/OB
    359364
    360   ippjython loader.py -test $objectqueuename once -simtest < /dev/null
     365  ippjython loader.py $debug -test $objectqueuename once -simtest < /dev/null
    361366  if ($status) exit
    362367endif
    … …  
    401406  echo "2000-01-01 00:00:00.0 " >> stackbatch.dat # trange_start
    402407  echo "2020-01-01 00:00:00.0 " >> stackbatch.dat # trange_end
    403   ippjython queue.py -test edit -simtest < stackbatch.dat
     408  ippjython queue.py $debug -test edit -simtest < stackbatch.dat
    404409  if ($status) exit
    405410endif
    … …  
    410415
    411416  rm -rf $OUTDIR/ST
    412   ippjython loader.py -test $stackqueuename once -simtest < /dev/null
     417  ippjython loader.py $debug -test $stackqueuename once -simtest < /dev/null
    413418  if ($status) exit
    414419endif
    … …  
    453458  echo "2000-01-01 00:00:00.0 " >> forcebatch.dat # trange_start
    454459  echo "2020-01-01 00:00:00.0 " >> forcebatch.dat # trange_end
    455   ippjython queue.py -test edit -simtest < forcebatch.dat
     460  ippjython queue.py $debug -test edit -simtest < forcebatch.dat
    456461  if ($status) exit 1
    457462endif
    … …  
    462467
    463468  rm -rf $OUTDIR/FW
    464   ippjython loader.py -test $forcedwarpqueuename once -simtest < /dev/null
     469  ippjython loader.py $debug -test $forcedwarpqueuename once -simtest < /dev/null
    465470  if ($status) exit
    466471endif
    … …  
    505510  echo "2000-01-01 00:00:00.0 " >> forcedobjectbatch.dat # trange_start
    506511  echo "2020-01-01 00:00:00.0 " >> forcedobjectbatch.dat # trange_end
    507   ippjython queue.py -test edit once -simtest < forcedobjectbatch.dat
     512  ippjython queue.py $debug -test edit once -simtest < forcedobjectbatch.dat
    508513  if ($status) exit
    509514endif
    … …  
    515520  rm -rf $OUTDIR/LO
    516521
    517   ippjython loader.py -test $forcedobjqueuename once -simtest < /dev/null
    518   if ($status) exit
    519 endif
    520 
     522  ippjython loader.py $debug -test $forcedobjqueuename once -simtest < /dev/null
     523  if ($status) exit
     524endif
     525

  • trunk/ippToPsps/test/mkgpc1data.dvo

    r38974 r38990  
    194194  end
    195195
    196   echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
    197   exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     196  if ($SMALLTEST)
     197    echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     198    exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     199  else
     200    echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     201    exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     202  end
    198203
    199204  if ($PARALLEL)
    … …  
    254259  end
    255260
    256 # echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
    257 # exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
    258 
    259   echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
    260   exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     261  if ($SMALLTEST)
     262    echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     263    exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     264  else
     265    echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     266    exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     267  end
    261268
    262269  if ($PARALLEL)
    … …  
    323330  end
    324331
    325   echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
    326   exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     332  if ($SMALLTEST)
     333    echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     334    exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     335  else
     336    echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     337    exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     338  end
    327339
    328340  echo dvolens -D CATDIR $catdir -region 8 12 18 22 -update-objects -update
Note: See TracChangeset for help on using the changeset viewer.