IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 39015


Ignore:
Timestamp:
Oct 30, 2015, 5:01:15 PM (11 years ago)
Author:
watersc1
Message:

Updates to make DF batches work. Test suite changes to implement that. There are still a few issues with header keywords not being respected completely, but I'm not fully sure if that's due to the simulated data or not. Change to fulltest.sh to move dbuser/pass into variables that can be changed easily.

Location:
trunk/ippToPsps
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/jython/diffbatch.py

    r38999 r39015  
    114114
    115115           self.difftypeID[num] = meta[1];
    116            self.ippDiffID[num] = stage_id
     116           self.ippDiffID[num] = stageid
    117117           self.posImageID[num] = meta[2];
    118118           self.negImageID[num] = meta[3];
    … …  
    122122           ## MEH -- should be able to use diff header now for mjd (see below)
    123123           #self.obsTime[num] = meta[7];
    124            self.expID[num] = meta[8];
     124           # tess_id is handled below, that's #8.
     125           self.expID[num] = meta[9];
    125126           self.logger.infoPair("loading diff stage cmf", diffSkyFileID)
    126127           self.fits[num] = gpc1Db.getDiffStageCmf(skychunk.dvoLabel, int(diffSkyFileID))
    … …  
    160161           self.dropTableVerbose("DiffToImage_"+str(num))
    161162           self.dropTableVerbose("DiffDetection_"+str(num))
     163           self.dropTableVerbose("DiffDetEffMeta_"+str(num))
    162164
    163165           if not self.analysisVer[num]: self.analysisVer[num] = -999
    … …  
    440442
    441443
     444    '''
     445    Populate DiffDetEffMeta
     446    '''
     447    def populateDiffDetEffMeta(self,num):
     448        diffSkyFileID = self.number[num]
     449        self.logger.infoPair("Processing table", "DiffDetEffMeta")
     450
     451        if diffSkyFileID <= 0:
     452            self.logger.infoPair("no diff data for number", num)
     453            return False
     454
     455        cmfTable = "SkyChip_deteff_" + str(num)
     456        if not cmfTable in self.tablesLoaded:
     457            self.logger.infoPair("no deteff data for diff", diffSkyFileID)
     458            self.logger.infoPair("(did not find table in tablesLoaded)", cmfTable)
     459            for name in self.tablesLoaded:
     460                print "-- table: " + name
     461            os._exit(3)
     462            return False
     463
     464        self.logger.infoPair("Found cmfTable:", cmfTable)
     465
     466        ippTableName = "DiffDetEffMeta_" + str(num)
     467        sql = "CREATE TABLE " + ippTableName + " LIKE DiffDetEffMeta"
     468        try: self.scratchDb.execute(sql)
     469        except: pass
     470
     471        self.logger.infoPair("Created tmp table:", ippTableName)
     472
     473        header = self.header[num]
     474        # CZW: this is broken, and can't get the zeropoint
     475        #        zpImage = self.scratchDb.getImageZeroPoint(diffSkyFileID)
     476        zpImage = 0.0
     477
     478        if zpImage < 0.0:
     479            zpImage = 0.0
     480        if zpImage == 0.0:
     481            self.logger.infoPair("CZW If this value for the zeropoint is zero, the getImageZeroPoint call is likely still broken", str(zpImage))
     482
     483
     484        magref = self.getKeyFloat(header,"%.8f","DETEFF.MAGREF") + zpImage
     485        self.logger.infoPair("Found zeropoint and set magref", str(magref))
     486        self.logger.infoPair("CZW If that magref is -5.804804, then it's reading from the PHU, not the Deteff extension.","In a test, I mean.")
     487        sqlLine = sqlUtility("INSERT INTO " + ippTableName + "(")
     488        sqlLine.group("diffImageID", str(diffSkyFileID))
     489        sqlLine.group("magref",       magref)
     490        sql = sqlLine.make(") VALUES ( ", ")")
     491           
     492
     493        try: self.scratchDb.execute(sql)
     494        except:
     495            self.logger.errorPair('failed sql: ', sql)
     496            raise
     497
     498        self.logger.infoPair("CZW:", "About to loop over deteff bins.")
     499
     500        for i in xrange(1, 14):
     501            stringint = str(i)
     502            if i < 10:
     503                stringint = '0' + str(i)
     504               
     505            sql = "UPDATE " + ippTableName + " , " + cmfTable + \
     506                " SET offset"+stringint + " = OFFSET, counts" \
     507                +stringint + " = COUNTS, diffMean" \
     508                +stringint + " = DIFF_MEAN, diffStdev" \
     509                +stringint + " = DIFF_STDEV, errMean" \
     510                +stringint + " = ERR_MEAN WHERE table_index = " + str(i)
     511               
     512            try: self.scratchDb.execute(sql)
     513            except:
     514                self.logger.errorPair('filed sql: ', sql)
     515                raise
     516            # end for
     517                # Append this like this?
     518        self.tablesToExport.append(ippTableName)
     519        # end filters
     520
     521
     522        return True
    442523
    443524
    … …  
    452533        self.logger.debug("Altering PSPS tables")
    453534        self.logger.debug("Creating indexes on PSPS tables")
    454         self.scratchDb.makeColumnUnique("DiffDetection", "objID")
     535        self.scratchDb.makeColumnUnique("DiffDetection", "diffObjID")
    455536        self.scratchDb.makeColumnPrimaryKey("DiffDetection", "ippDetectID")
    456537
    … …  
    555636            self.setMinMaxDiffObjID(["DiffDetection_"+str(num)])
    556637
    557 
     638            self.populateDiffDetEffMeta(num)
     639            self.logger.infoPair("populated","DiffDetEffMeta_"+str(num))
     640            # CZW: I appended to tablesToExport inside populateDiffDetEffMeta to be like stack.
    558641        #test = self.scratchDb.getRowCount("Object")
    559642        #return False
    … …  
    623706                        self.haveApNpix[num] = 1
    624707
    625                 myTable = table.name + "_" + str(num)
     708
     709                myTable = table.name.replace(".","_") + "_" + str(num)
    626710                count = count + 1
    627711                     

  • trunk/ippToPsps/jython/gpc1db.py

    r38973 r39015  
    433433    def getDiffStageMeta(self, diffSkyfileID):
    434434
    435         self.logger.debug("Querying GPC1 for stack meta data")
     435        self.logger.debug("Querying GPC1 for diff meta data")
    436436
    437437        meta = []
    … …  
    528528               JOIN diffInputSkyfile on (diff_skyfile_id = stage_extra1 \
    529529               AND diff_id = stage_id AND stage = 'diff') \
    530                JOIN diffSkyfile using (diff_id, skycell_id) \
     530               JOIN diffSkyfile using (diff_id, skycell_id, diff_skyfile_id) \
    531531               WHERE diff_skyfile_id = " + str(diffSkyfileID) + " \
    532532               AND minidvodbRun.state = 'merged' \
    … …  
    557557                files.append(i.rstrip())
    558558                print "append"
    559         print i
     559# No clue what this is here for, but it breaks things.               
     560#        print i
     561        else:
     562            files = glob.glob(pathBase + ".cmf")
     563
    560564        if len(files) < 1: return None
    561565        fits = Fits(self.logger, self.config, files[0])

  • trunk/ippToPsps/test/fulltest.sh

    r39009 r39015  
    11#!/bin/csh -f
     2
     3set dbuser = 'dvo'
     4set dbpass = '-pdvo'
     5# set dbuser = 'czw'
     6# set dbpass = '-pczw'
    27
    38chmod +x mkgpc1data.dvo mkgpc1.sh
    … …  
    2934set forcedobjqueue = 1
    3035set forcedobjbatch = 1
     36set diffqueue    = 1
     37set diffbatch    = 1
     38set diffobjqueue = 1
     39set diffobjbatch = 1
     40
    3141
    3242set options = "-test"
    … …  
    4252    echo " -XX : turn off features"
    4353    echo " +XX : turn on features"
    44     echo "  features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object, forcedobject"
     54    echo "  features: all, mkgpc1, initdb, initscratch, initbatch, camera, stack, forcedwarp, object, forcedobject, diff, diffobj"
    4555    exit 2
    4656  case -all
    … …  
    5969    set forcedobjqueue = 0
    6070    set forcedobjbatch = 0
     71    set diffqueue = 0
     72    set diffbatch = 0
     73    set diffobjqueue = 0
     74    set diffobjbatch = 0
    6175    breaksw;
    6276  case -mkgpc1
    … …  
    91105    set forcedobjqueue = 0
    92106    set forcedobjbatch = 0
     107    breaksw;
     108  case -diff
     109    set diffqueue = 0
     110    set diffbatch = 0
     111    breaksw;
     112  case -diffobj
     113    set diffobjqueue = 0
     114    set diffobjbatch = 0
    93115    breaksw;
    94116  case -debug
    … …  
    125147    set forcedobjqueue = 1
    126148    set forcedobjbatch = 1
     149    set diffqueue = 1
     150    set diffbatch = 1
     151    set diffobjqueue = 1
     152    set diffobjbatch = 1
    127153    breaksw;
    128154  case +mkgpc1
    … …  
    157183    set forcedobjqueue = 1
    158184    set forcedobjbatch = 1
     185    breaksw;
     186  case +diff
     187    set diffqueue = 1
     188    set diffbatch = 1
     189    breaksw;
     190  case +diffobj
     191    set diffobjqueue = 1
     192    set diffobjbatch = 1
    159193    breaksw;
    160194  default:
    … …  
    169203    echo " -XX : turn off features"
    170204    echo " +XX : turn on features"
    171     echo "  features: all, mkgpc1, initdb, initscratch, initbatch, camera, object, stack, forcedwarp, forcedobject"
     205    echo "  features: all, mkgpc1, initdb, initscratch, initbatch, camera, object, stack, forcedwarp, forcedobject, diff, diffobj"
    172206    exit 2
    173207endif
    … …  
    179213set forcedwarpqueuename = $queuename\_wrp
    180214set forcedobjqueuename = $queuename\_fwo
     215set diffqueuename = $queuename\_dif
     216set diffobjqueuename = $queuename\_dio
     217
    181218
    182219if ($mkgpc1) then
    … …  
    197234  # does the database already exist?
    198235  # XXX this command will blow away an existing database!!
    199   mysql -h localhost -u dvo -pdvo -e "create database if not exists ipptopsps_test" || exit 3
    200   mysql -h localhost -u dvo -pdvo ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql || exit 3
     236  mysql -h localhost -u $dbuser $dbpass -e "create database if not exists ipptopsps_test" || exit 3
     237  mysql -h localhost -u $dbuser $dbpass ipptopsps_test < $IPPTOPSPS_DATA/ippToPspsDbSchema.sql || exit 3
    201238  echo " ---- init ipptopsps_test: done ----"
    202239endif
    … …  
    208245
    209246  # create the database (ignore failure if it already exists)
    210   mysql -h localhost -u dvo -pdvo -e "drop database if exists ipptopsps_test_scratch"
    211   mysql -h localhost -u dvo -pdvo -e "create database ipptopsps_test_scratch"
     247  mysql -h localhost -u $dbuser $dbpass -e "drop database if exists ipptopsps_test_scratch"
     248  mysql -h localhost -u $dbuser $dbpass -e "create database ipptopsps_test_scratch"
    212249  echo ippjython setupScratchDb.py $options
    213250       ippjython setupScratchDb.py $options < /dev/null
    … …  
    527564endif
    528565
     566
     567############################ DF batches ################################
     568
     569# this code is sensitive to a previously existing database?
     570if ($diffqueue) then
     571  echo ""
     572  echo " ---- queue DF batches ----"
     573
     574  echo "$diffqueuename  " >  diffbatch.dat # skychunk name
     575  echo "test_ds               " >> diffbatch.dat # datastore_product
     576  echo "IPP_PSPS              " >> diffbatch.dat # datastore_type
     577  echo "0                     " >> diffbatch.dat # datastore_publish
     578  echo "catdir.dif            " >> diffbatch.dat # dvo_label
     579  echo "$OUTDIR/catdir.dif    " >> diffbatch.dat # dvo_location
     580  echo "9                     " >> diffbatch.dat # min_ra
     581  echo "11                    " >> diffbatch.dat # max_ra
     582  echo "19                    " >> diffbatch.dat # min_dec
     583  echo "21                    " >> diffbatch.dat # max_dec
     584  echo "2                     " >> diffbatch.dat # box_size
     585  echo "$OUTDIR               " >> diffbatch.dat # base_path
     586  echo "0                     " >> diffbatch.dat # data_release
     587  echo "0                     " >> diffbatch.dat # delete_local
     588  echo "1                     " >> diffbatch.dat # delete_datastore
     589  echo "1                     " >> diffbatch.dat # delete_dxlayer
     590  echo "2012-12-10 00:00:00.0 " >> diffbatch.dat # epoch
     591  echo "3PI                   " >> diffbatch.dat # survey
     592  echo "3PI                   " >> diffbatch.dat # psps_survey
     593  echo "0                     " >> diffbatch.dat # queue_P2
     594  echo "0                     " >> diffbatch.dat # queue_ST
     595  echo "0                     " >> diffbatch.dat # queue_OB
     596  echo "1                     " >> diffbatch.dat # queue_DF
     597  echo "0                     " >> diffbatch.dat # queue_DO
     598  echo "0                     " >> diffbatch.dat # queue_FW
     599  echo "0                     " >> diffbatch.dat # queue_FO
     600  echo "1                     " >> diffbatch.dat # active
     601  echo "0                     " >> diffbatch.dat # parallel
     602  echo "use_new               " >> diffbatch.dat # P2_smf_version
     603  echo "any                   " >> diffbatch.dat # ST_cmf_version
     604  echo "2000-01-01 00:00:00.0 " >> diffbatch.dat # trange_start
     605  echo "2020-01-01 00:00:00.0 " >> diffbatch.dat # trange_end
     606  ippjython queue.py $options edit -simtest < diffbatch.dat
     607  if ($status) exit 1
     608endif
     609
     610if ($diffbatch) then
     611  echo ""
     612  echo " ---- load DF batches ----"
     613
     614  rm -rf $OUTDIR/FW
     615  ippjython loader.py $options $diffqueuename once -simtest < /dev/null
     616  if ($status) exit
     617endif
     618
     619############################ DO batches ################################
     620
     621# this code is sensitive to a previously existing database?
     622if ($diffobjqueue) then
     623  echo ""
     624  echo " ---- queue do batches ----"
     625
     626  echo "$diffobjqueuename   " >  diffobjbatch.dat # skychunk name
     627  echo "test_ds               " >> diffobjbatch.dat # datastore_product
     628  echo "IPP_PSPS              " >> diffobjbatch.dat # datastore_type
     629  echo "0                     " >> diffobjbatch.dat # datastore_publish
     630  echo "catdir.dif            " >> diffobjbatch.dat # dvo_label
     631  echo "$OUTDIR/catdir.dif    " >> diffobjbatch.dat # dvo_location
     632  echo "9                     " >> diffobjbatch.dat # min_ra
     633  echo "11                    " >> diffobjbatch.dat # max_ra
     634  echo "19                    " >> diffobjbatch.dat # min_dec
     635  echo "21                    " >> diffobjbatch.dat # max_dec
     636  echo "2                     " >> diffobjbatch.dat # box_size
     637  echo "$OUTDIR               " >> diffobjbatch.dat # base_path
     638  echo "0                     " >> diffobjbatch.dat # data_release
     639  echo "0                     " >> diffobjbatch.dat # delete_local
     640  echo "1                     " >> diffobjbatch.dat # delete_datastore
     641  echo "1                     " >> diffobjbatch.dat # delete_dxlayer
     642  echo "2012-12-10 00:00:00.0 " >> diffobjbatch.dat # epoch
     643  echo "3PI                   " >> diffobjbatch.dat # survey
     644  echo "3PI                   " >> diffobjbatch.dat # psps_survey
     645  echo "0                     " >> diffobjbatch.dat # queue_P2
     646  echo "0                     " >> diffobjbatch.dat # queue_ST
     647  echo "0                     " >> diffobjbatch.dat # queue_OB
     648  echo "0                     " >> diffobjbatch.dat # queue_DF
     649  echo "1                     " >> diffobjbatch.dat # queue_DO
     650  echo "0                     " >> diffobjbatch.dat # queue_FW
     651  echo "0                     " >> diffobjbatch.dat # queue_FO
     652  echo "1                     " >> diffobjbatch.dat # active
     653  echo "0                     " >> diffobjbatch.dat # parallel
     654  echo "use_new               " >> diffobjbatch.dat # P2_smf_version
     655  echo "any                   " >> diffobjbatch.dat # ST_cmf_version
     656  echo "2000-01-01 00:00:00.0 " >> diffobjbatch.dat # trange_start
     657  echo "2020-01-01 00:00:00.0 " >> diffobjbatch.dat # trange_end
     658  ippjython queue.py $options edit once -simtest < diffobjbatch.dat
     659  if ($status) exit
     660endif
     661
     662if ($diffobjbatch) then
     663  echo ""
     664  echo " ---- load LO batches ----"
     665
     666  rm -rf $OUTDIR/LO
     667
     668  ippjython loader.py $options $diffobjqueuename once -simtest < /dev/null
     669  if ($status) exit
     670endif
     671

  • trunk/ippToPsps/test/gpc1schema.sql

    r38948 r39015  
    11
    22  CREATE TABLE ippToPspsFake       ( my_id INT  );
    3   CREATE TABLE skycell             ( skycell_id CHAR(64), radeg REAL, decdeg REAL );
     3  CREATE TABLE skycell             ( tess_id CHAR(64), skycell_id CHAR(64), radeg REAL, decdeg REAL );
    44  CREATE TABLE rawExp              ( exp_id INT, exp_name CHAR(64), exp_time REAL, filter CHAR(64), bg REAL, bg_stdev REAL, ra FLOAT, decl FLOAT, dateobs DATETIME );
    55  CREATE TABLE chipRun             ( exp_id INT, chip_id INT );
    … …  
    1919  CREATE TABLE skycalResult        ( skycal_id INT, path_base CHAR(255));
    2020
     21  CREATE TABLE diffRun             ( diff_id INT, tess_id CHAR(64), state CHAR(16), diff_mode INT, software_ver CHAR(16) );
     22  CREATE TABLE diffInputSkyfile    ( diff_id INT, warp1 INT, stack2 INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64) );
     23  CREATE TABLE diffSkyfile         ( diff_id INT, diff_skyfile_id INT, tess_id CHAR(64), skycell_id CHAR(64), path_base CHAR(255) );
     24
    2125  CREATE TABLE addRun              ( add_id INT, stage CHAR(64), stage_extra1 INT, state CHAR(64), stage_id INT, minidvodb_name CHAR(64) );
    2226  CREATE TABLE addProcessedExp     ( add_id INT, fault INT );

  • trunk/ippToPsps/test/mkgpc1.sh

    r38964 r39015  
    5858  set ntable = `mysql -B -h $dbhost -u $dbuser $dbpass $dbname -e "show tables" | wc -l`
    5959  echo ntable: $ntable
    60   if ($ntable != 25) then
    61     echo "expected 25 lines (24 tables), but got $ntable, not deleting"
     60  if ($ntable != 28) then
     61    echo "expected 28 lines (27 tables), but got $ntable, not deleting"
    6262    exit 2
    6363  endif

  • trunk/ippToPsps/test/mkgpc1data.dvo

    r38996 r39015  
    2323# imagedata describes the full fake exposure set.  the fields are
    2424list imagedata
    25 # exp_id stk_id filebase         M_off date       time      warpfile         filter
    26    1     1      $OUTDIR/test.01  0.000 2010/01/01 01:00:00  $OUTDIR/warp.01  g
    27    2     1      $OUTDIR/test.02 -0.025 2010/01/01 02:00:00  $OUTDIR/warp.02  g
    28    3     1      $OUTDIR/test.03  0.025 2010/01/01 03:00:00  $OUTDIR/warp.03  g
    29    4     1      $OUTDIR/test.04  0.010 2010/01/01 04:00:00  $OUTDIR/warp.04  g
    30    5     2      $OUTDIR/test.05  0.000 2010/01/01 05:00:00  $OUTDIR/warp.05  g
    31    6     2      $OUTDIR/test.06 -0.025 2010/01/01 06:00:00  $OUTDIR/warp.06  g
    32    7     2      $OUTDIR/test.07  0.025 2010/01/01 07:00:00  $OUTDIR/warp.07  g
    33    8     2      $OUTDIR/test.08  0.010 2010/01/01 08:00:00  $OUTDIR/warp.08  g
    34    9     3      $OUTDIR/test.09  0.000 2010/01/01 09:00:00  $OUTDIR/warp.09  g
    35   10     3      $OUTDIR/test.10 -0.025 2010/01/01 10:00:00  $OUTDIR/warp.10  g
    36   11     3      $OUTDIR/test.11  0.025 2010/01/01 11:00:00  $OUTDIR/warp.11  g
    37   12     3      $OUTDIR/test.12  0.010 2010/01/01 12:00:00  $OUTDIR/warp.12  g
    38   13     4      $OUTDIR/test.13 -0.025 2010/01/02 01:00:00  $OUTDIR/warp.13  r
    39   14     4      $OUTDIR/test.14  0.025 2010/01/02 02:00:00  $OUTDIR/warp.14  r
    40   15     4      $OUTDIR/test.15  0.010 2010/01/02 03:00:00  $OUTDIR/warp.15  r
    41   16     4      $OUTDIR/test.16  0.000 2010/01/02 04:00:00  $OUTDIR/warp.16  r
    42   17     5      $OUTDIR/test.17 -0.025 2010/01/02 05:00:00  $OUTDIR/warp.17  r
    43   18     5      $OUTDIR/test.18  0.025 2010/01/02 06:00:00  $OUTDIR/warp.18  r
    44   19     5      $OUTDIR/test.19  0.010 2010/01/02 07:00:00  $OUTDIR/warp.19  r
    45   20     5      $OUTDIR/test.20  0.000 2010/01/02 08:00:00  $OUTDIR/warp.20  r
    46   21     6      $OUTDIR/test.21 -0.025 2010/01/02 09:00:00  $OUTDIR/warp.21  r
    47   22     6      $OUTDIR/test.22  0.025 2010/01/02 10:00:00  $OUTDIR/warp.22  r
    48   23     6      $OUTDIR/test.23  0.010 2010/01/02 11:00:00  $OUTDIR/warp.23  r
    49   24     6      $OUTDIR/test.24 -0.020 2010/01/02 12:00:00  $OUTDIR/warp.24  r
     25# exp_id stk_id filebase         M_off date       time      warpfile         filter  difffile
     26   1     1      $OUTDIR/test.01  0.000 2010/01/01 01:00:00  $OUTDIR/warp.01  g       $OUTDIR/diff.01
     27   2     1      $OUTDIR/test.02 -0.025 2010/01/01 02:00:00  $OUTDIR/warp.02  g       $OUTDIR/diff.02
     28   3     1      $OUTDIR/test.03  0.025 2010/01/01 03:00:00  $OUTDIR/warp.03  g       $OUTDIR/diff.03
     29   4     1      $OUTDIR/test.04  0.010 2010/01/01 04:00:00  $OUTDIR/warp.04  g       $OUTDIR/diff.04
     30   5     2      $OUTDIR/test.05  0.000 2010/01/01 05:00:00  $OUTDIR/warp.05  g       $OUTDIR/diff.05
     31   6     2      $OUTDIR/test.06 -0.025 2010/01/01 06:00:00  $OUTDIR/warp.06  g       $OUTDIR/diff.06
     32   7     2      $OUTDIR/test.07  0.025 2010/01/01 07:00:00  $OUTDIR/warp.07  g       $OUTDIR/diff.07
     33   8     2      $OUTDIR/test.08  0.010 2010/01/01 08:00:00  $OUTDIR/warp.08  g       $OUTDIR/diff.08
     34   9     3      $OUTDIR/test.09  0.000 2010/01/01 09:00:00  $OUTDIR/warp.09  g       $OUTDIR/diff.09
     35  10     3      $OUTDIR/test.10 -0.025 2010/01/01 10:00:00  $OUTDIR/warp.10  g       $OUTDIR/diff.10
     36  11     3      $OUTDIR/test.11  0.025 2010/01/01 11:00:00  $OUTDIR/warp.11  g       $OUTDIR/diff.11
     37  12     3      $OUTDIR/test.12  0.010 2010/01/01 12:00:00  $OUTDIR/warp.12  g       $OUTDIR/diff.12
     38  13     4      $OUTDIR/test.13 -0.025 2010/01/02 01:00:00  $OUTDIR/warp.13  r       $OUTDIR/diff.13
     39  14     4      $OUTDIR/test.14  0.025 2010/01/02 02:00:00  $OUTDIR/warp.14  r       $OUTDIR/diff.14
     40  15     4      $OUTDIR/test.15  0.010 2010/01/02 03:00:00  $OUTDIR/warp.15  r       $OUTDIR/diff.15
     41  16     4      $OUTDIR/test.16  0.000 2010/01/02 04:00:00  $OUTDIR/warp.16  r       $OUTDIR/diff.16
     42  17     5      $OUTDIR/test.17 -0.025 2010/01/02 05:00:00  $OUTDIR/warp.17  r       $OUTDIR/diff.17
     43  18     5      $OUTDIR/test.18  0.025 2010/01/02 06:00:00  $OUTDIR/warp.18  r       $OUTDIR/diff.18
     44  19     5      $OUTDIR/test.19  0.010 2010/01/02 07:00:00  $OUTDIR/warp.19  r       $OUTDIR/diff.19
     45  20     5      $OUTDIR/test.20  0.000 2010/01/02 08:00:00  $OUTDIR/warp.20  r       $OUTDIR/diff.20
     46  21     6      $OUTDIR/test.21 -0.025 2010/01/02 09:00:00  $OUTDIR/warp.21  r       $OUTDIR/diff.21
     47  22     6      $OUTDIR/test.22  0.025 2010/01/02 10:00:00  $OUTDIR/warp.22  r       $OUTDIR/diff.22
     48  23     6      $OUTDIR/test.23  0.010 2010/01/02 11:00:00  $OUTDIR/warp.23  r       $OUTDIR/diff.23
     49  24     6      $OUTDIR/test.24 -0.020 2010/01/02 12:00:00  $OUTDIR/warp.24  r       $OUTDIR/diff.24
    5050end
    5151
    … …  
    6565  mkcatdir.stk PS1_V5 PS1_V5
    6666  mkcatdir.wrp PS1_V5_Lensing PS1_V5
     67  mkcatdir.dif PS1_DV5 PS1_V5
    6768  insert.stack.set
    6869end
    … …  
    7273  $dbhost = localhost
    7374  $dbname = gpc1test
    74   $dbuser = dvo
    75   $dbpass = dvo
     75#  $dbuser = dvo
     76#  $dbpass = dvo
     77  $dbuser = czw
     78  $dbpass = czw
    7679
    7780  exec mkgpc1.sh delete $dbhost $dbname $dbuser $dbpass
    … …  
    8285  # system-wide info
    8386  dbinsert ippToPspsFake (my_id) values (2)
    84   dbinsert skycell (radeg, decdeg, skycell_id) values ($RA, $DEC, $SKYCELL)
     87  dbinsert skycell (radeg, decdeg, tess_id, skycell_id) values ($RA, $DEC, $TESS_ID, $SKYCELL)
    8588end
    8689
    … …  
    477480end
    478481
     482macro mkcatdir.dif
     483  if ($0 != 3)
     484      echo "mkcatdir.cam (cmftype) (dvotype)"
     485      break
     486  end
     487
     488  # tapPLAN 4
     489
     490  mkdir $OUTDIR
     491
     492  local i catdir ID stkID rawfile cmffile offset
     493  $catdir = $OUTDIR/catdir.dif
     494  exec rm -rf $catdir
     495
     496  $TIMEFORMAT = mjd
     497  $TIMEREF = 2001/01/01,00:00:00
     498
     499  for i 0 $imagedata:n
     500     list word -split $imagedata:$i
     501     $ID = $word:0
     502     $stkID = $word:1
     503     $rawfile = $word:2\.txt
     504     $cmffile = $word:8\.cmf
     505     $offset = $word:3
     506     sprintf myDATE "%s" $word:4
     507     sprintf myTIME "%s" $word:5
     508     $filter = $word:7
     509
     510     if ($SMALLTEST && ($stkID != 1) && ($stkID != 4)) continue
     511
     512     echo ctimes -abs $myDATE,$myTIME -var mjd
     513     ctimes -abs $myDATE,$myTIME -var mjd
     514
     515    # create an empty header and populate with the desired keywords
     516    mcreate dummy 0 0
     517    keyword dummy MJD-OBS  -wf $mjd
     518    keyword dummy FILTERID -w  $filter.00000
     519    keyword dummy EXPTIME  -wf 1.0
     520
     521#    keyword dummy ZPT_ERR  -wf 0.01
     522#    keyword dummy EXPREQ   -wf 1.0
     523    keyword dummy AIRMASS  -wf 1.0
     524    keyword dummy RA       -wf $RA
     525    keyword dummy DEC      -wf $DEC
     526    keyword dummy CTYPE1   -w "RA---DIS"
     527    keyword dummy CTYPE2   -w "DEC--DIS"
     528    keyword dummy CRVAL1   -wf $RA
     529    keyword dummy CRVAL2   -wf $DEC
     530    keyword dummy CRPIX1   -wf 0.0
     531    keyword dummy CRPIX2   -wf 0.0
     532    keyword dummy CDELT1   -wf {1.0/3600}
     533    keyword dummy CDELT2   -wf {1.0/3600}
     534    keyword dummy PC001001 -wf 1.0
     535    keyword dummy PC001002 -wf 0.0
     536    keyword dummy PC002001 -wf 0.0
     537    keyword dummy PC002002 -wf 1.0
     538    keyword dummy NPLYTERM -wd 0
     539    keyword dummy PCA1X3Y0 -wf 0.0
     540    keyword dummy PCA1X2Y1 -wf 0.0
     541    keyword dummy PCA1X1Y2 -wf 0.0
     542    keyword dummy PCA1X0Y3 -wf 0.0
     543    keyword dummy PCA1X2Y0 -wf 0.0
     544    keyword dummy PCA1X1Y1 -wf 0.0
     545    keyword dummy PCA1X0Y2 -wf 0.0
     546    keyword dummy PCA2X3Y0 -wf 0.0
     547    keyword dummy PCA2X2Y1 -wf 0.0
     548    keyword dummy PCA2X1Y2 -wf 0.0
     549    keyword dummy PCA2X0Y3 -wf 0.0
     550    keyword dummy PCA2X2Y0 -wf 0.0
     551    keyword dummy PCA2X1Y1 -wf 0.0
     552    keyword dummy PCA2X0Y2 -wf 0.0
     553
     554    $zp = $offset + 23.0
     555    keyword dummy "HIERARCH FPA.ZP" -wf $zp
     556    keyword dummy "HIERARCH DETEFF.MAGREF" -wf -5.8048040
     557    wd dummy $cmffile
     558
     559    mkinput $offset $rawfile
     560    echo mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC
     561    exec mkcmf -diff -photcode SIMTEST.$filter.SkyChip -no-noise -extroot SkyChip -append $rawfile $cmffile -date $myDATE -time $myTIME -mjd $mjd -radec $RA $DEC -type $1 -imageID $ID -sourceID 36 -bad-psfqf-frac $BAD_PSFQF_FRAC
     562
     563
     564    insert.dif $ID $ID $stkID $word:8
     565
     566    if ($i == 3)
     567      echo "NOTE: not adding image $i to DVO : this simulates images with poor astrometry"
     568      continue
     569    end
     570    echo addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
     571    exec addstar -D SKY_DEPTH 4 -D CATDIR $catdir -D CAMERA simtest $cmffile -D CATFORMAT $2 -quick-airmass
     572  end
     573  if ($SMALLTEST)
     574    echo relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     575    exec relphot -D CATDIR $catdir -region 8 12 18 22 -averages -update
     576  else
     577    echo relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     578    exec relphot -D CATDIR $catdir -region 8 12 18 22 -images g,r,i,z,y -update
     579  end
     580
     581  if ($PARALLEL)
     582    $hostname = `hostname`
     583    exec rm -f $catdir/HostTable.dat
     584    output $catdir/HostTable.dat
     585    echo "# ID Hostname Catdir"
     586    echo " 1 $hostname $catdir.p1"
     587    echo " 2 $hostname $catdir.p2"
     588    echo " 3 $hostname $catdir.p3"
     589    output stdout
     590    exec dvodist -out $catdir
     591  end
     592
     593  # for i 0 $offset:n
     594  #   tapOK {abs(Mcal[$i] - Mcal[0] - $offset:$i) < 0.001} "Mcal $i"
     595  # end
     596
     597  # tapDONE
     598end
     599
    479600
    480601macro insert.stack.set
    … …  
    516637  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (3, 3, "full", "catdir.wrp")
    517638  dbinsert mergedvodbProcessed (merge_id, fault) values (3, 0)
    518 end
     639
     640  # catdirs for the diffs
     641  dbinsert minidvodbRun (minidvodb_name, minidvodb_id, state) values ("catdir.dif", 4, "merged")
     642  dbinsert minidvodbProcessed (minidvodb_id, fault) values (4, 0)
     643
     644  dbinsert mergedvodbRun (minidvodb_id, merge_id, state, mergedvodb) values (4, 4, "full", "catdir.dif")
     645  dbinsert mergedvodbProcessed (merge_id, fault) values (4, 0)
     646
     647end
     648
     649macro insert.dif
     650  if ($0 != 5)
     651      echo "USAGE: insert.dif (diffID) (warpID) (stackID) (filename)"
     652      break
     653  end
     654
     655  local difID wrpID stkID filename
     656  $difID = $1
     657  $wrpID = $2
     658  $stkID = $3
     659  $filename = $4
     660
     661  echo $myDATE
     662  echo $myTIME
     663
     664  $addID = $difID + 100
     665  $diffSkyfileID = $difID + 25;
     666  dbinsert addRun (add_id, stage, state, stage_id, stage_extra1, minidvodb_name) values ($addID, "diff", "full", $difID, $diffSkyfileID, "catdir.dif")
     667  dbinsert addProcessedExp (add_id, fault) values ($addID, 0)
     668
     669  dbinsert diffRun (diff_id, tess_id, state, diff_mode, software_ver) values ($difID, $TESS_ID, "full", 2, "38000M")
     670  dbinsert diffInputSkyfile (diff_id, warp1, stack2, diff_skyfile_id, tess_id, skycell_id) values ($difID, $wrpID, $stkID, $diffSkyfileID, $TESS_ID,  $SKYCELL )
     671  dbinsert diffSkyfile (diff_id, diff_skyfile_id, tess_id, skycell_id, path_base) values ($difID, $diffSkyfileID, $TESS_ID, $SKYCELL, "$filename")
     672#  dbinsert diffSkyfile (diff_id,
     673
     674# finish this up.
     675end
     676
    519677
    520678
Note: See TracChangeset for help on using the changeset viewer.