IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42603


Ignore:
Timestamp:
Jan 28, 2024, 2:24:26 PM (2 years ago)
Author:
eugene
Message:

working on xcff_warp.pl

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20230313/tools/eam/xcstack.20231124/make.xcstack.v1.sh

    r42599 r42603  
    101101    echo "insert into fakeRun     (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql
    102102    echo "insert into warpRun     (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
    103     echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql
     103    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/warp.$warp_id.$skycell', 'full');" >> test_$cam.sql
    104104    @ warp_id ++
    105105  end
    … …  
    110110  while ($nstack < $nwarps_max - $nwarps_min + 1)
    111111   
    112     echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
    113     echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
     112    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir/stacks.$cam', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
     113    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/stacks.$cam/stack.$stack_id.$skycell', 0.95, 0, 0);" >> test_$cam.sql
    114114   
    115115    @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
    … …  
    157157    echo "insert into fakeRun     (cam_id, fake_id) values ($warp_id, $warp_id);" >> test_$cam.sql
    158158    echo "insert into warpRun     (warp_id, fake_id, tess_id, state, workdir, label) values ($warp_id, $warp_id, 'Fake.Tess', 'full', '$mydir/warps.$cam', 'label.$cam');" >> test_$cam.sql
    159     echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/dummy.wrp.$warp_id.$skycell', 'full');" >> test_$cam.sql
     159    echo "insert into warpSkyfile (warp_id, skycell_id, tess_id, quality, fault, good_frac, path_base, data_state) values ($warp_id, '$skycell', 'Fake.Tess', 0, 0, 0.8, '$mydir/warps.$cam/warp.$warp_id.$skycell', 'full');" >> test_$cam.sql
    160160    @ warp_id ++
    161161  end
    … …  
    166166  while ($nstack < $nwarps_max - $nwarps_min + 1)
    167167   
    168     echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
    169     echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/$skycell.$cam', 0.95, 0, 0);" >> test_$cam.sql
     168    echo "insert into stackRun (stack_id, state, workdir, label, skycell_id, tess_id, filter) values ($stack_id, 'full', '$mydir/stacks.$cam', 'label.$nstack.$cam', '$skycell', 'Fake.Tess', '$filter');" >> test_$cam.sql
     169    echo "insert into stackSumSkyfile (stack_id, path_base, good_frac, fault, quality) values ($stack_id, '$mydir/stacks.$cam/stack.$stack_id.$skycell', 0.95, 0, 0);" >> test_$cam.sql
    170170   
    171171    @ warp_id_inp = ($nwarps_max * ($ngroup - 1) + 1)
    … …  
    203203while ($nstack < $nwarps_max - $nwarps_min + 1)
    204204  # define xcstacks by matching labels:
    205   xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1
     205  xcstacktool -dbname test_xcam -definebyquery -xcamera test_cam1 -set_workdir $mydir/xcstack -set_label Test.xcstk.$nstack -select_label label.$nstack.cam1
    206206
    207207  # we have to define the relationship between instack and xcstack filters, so both must be specified
    … …  
    220220set Tskycells = (`grep skycell_id xcstack.md | awk '{print $3}'`)
    221221set workdir = `grep workdir xcstack.md | awk '{print $3}' | head -n 1`
     222mkdir -p $workdir
    222223
    223224# pretend to run xcstack
    224225set n = 1
    225226while ($n <= $#stk_ids)
    226   xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n]
     227  # OUTROOT is defined in xcstack.pro based on workdir : "%s/%s/%s/%s.%s.stk.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCSTACK_ID
     228  if (0 && ($n == 1)) then
     229    # manually run xcstack : note the camera must match a real camera in ippconfig/system.conf
     230    mkdir -p $workdir/Fake.Tess/$Tskycells[$n]
     231    echo xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_ids[$n] --outroot $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] --run-state new --camera GPC1
     232    xcstack_skycell.pl --dbname test_xcam --xcstack_id $stk_ids[$n] --outroot $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] --run-state new --camera GPC1
     233  else
     234    xcstacktool -dbname test_xcam -addsumskyfile -path_base $workdir/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].stk.$stk_ids[$n] -hostname elepaio -dtime_script 45.9999994374812 -xcstack_id $stk_ids[$n]
     235  endif
    227236  @ n++
    228237end
    229238
    230 # exit 2
    231 
    232239echo "******** xcskytool steps (static sky) ********"
    233240
    … …  
    235242
    236243# do the first set as separate filters
    237 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r
    238 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g
     244xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter r
     245xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.0 -select_label Test.xcstk.0 -select_filter g
    239246
    240247# do the second and third sets as multi-filters
    241 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r
    242 xcskytool -dbname test_xcam -definebyquery -set_workdir here/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r
     248xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.1 -select_label Test.xcstk.1 -select_filter g -select_filter r
     249xcskytool -dbname test_xcam -definebyquery -set_workdir $mydir/xcsky -set_label Test.xcsky.2 -select_label Test.xcstk.2 -select_filter g -select_filter r
    243250
    244251xcskytool -dbname test_xcam -todo -label Test.xcsky.0
    … …  
    259266  set n = 1
    260267  while ($n <= $#sky_ids)
    261     xcskytool -dbname test_xcam -addresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n]
    262     xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n]
     268    # OUTROOT is defined in xcsky.pro based on workdir : "%s/%s/%s/%s.%s.sky.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKY_ID
     269    if (0 && ($n == 2)) then
     270      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
     271      echo xcsky.pl --dbname test_xcam --xcsky_id $sky_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] --camera GPC1
     272      xcsky.pl --dbname test_xcam --xcsky_id $sky_ids[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] --camera GPC1
     273    else
     274      xcskytool -dbname test_xcam -addresult -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].sky.$sky_ids[$n] -hostname elepaio -dtime_script 45 -dtime_phot 28 -sources 100 -num_inputs 1 -good_frac 0.5 -xcsky_id $sky_ids[$n]
     275      xcskytool -dbname test_xcam -result -xcsky_id $sky_ids[$n]
     276    endif
    263277    @ n++
    264278  end
    … …  
    281295# queue all three labels
    282296foreach label (0 1 2)
     297  # workdir is inherited from xcsky if not specified here
    283298  xcskytool -dbname test_xcam -definexccalrun -select_label Test.xcsky.$label -set_label Test.xccal.$label
    284299end
    285300
    286301# mysql> select xccal_id, xcsky_id, xcstack_id, xccalRun.label as calLabel, xcskyRun.label as skyLabel, xcstackRun.label as stkLabel, xcstackRun.filter, xcskyResult.path_base, xcstackSumSkyfile.path_base from xccalRun join xcskyInput using (xcsky_id, xcstack_id) join xcskyRun using (xcsky_id) join xcskyResult using (xcsky_id) join xcstackRun using (xcstack_id) join xcstackSumSkyfile using (xcstack_id);
    287 
    288302# NOTE: xccalRun inherits the label, data_group, dist_group, reduction from xcskyRun
    289303
    … …  
    291305  xcskytool -dbname test_xcam -pendingxccalrun -label Test.xccal.$label > xccal.md
    292306  set cal_ids   = (`grep xccal_id   xccal.md | awk '{print $3}'`)
     307  set stk_ids   = (`grep xcstack_id xccal.md | awk '{print $3}'`)
    293308  set Tskycells = (`grep skycell_id xccal.md | awk '{print $3}'`)
    294309  set workdir   = (`grep workdir    xccal.md | awk '{print $3}'`)
     310  set pathbase  = (`grep path_base  xccal.md | awk '{print $3}'`)
    295311 
    296312  ## pretend to run xcsky.pl
    297313  set n = 1
    298314  while ($n <= $#cal_ids)
    299     xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[1]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n]
    300     xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n]
     315    # OUTROOT is defined in xccal.pro based on workdir : "%s/%s/%s/%s.%s.cal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $SKYCAL_ID  ****** TBD *****
     316    if (0 && ($n == 1)) then
     317      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
     318      echo xccal.pl --dbname test_xcam --xccal_id $cal_ids[$n] --xcstack_id $stk_ids[$n] --path_base $pathbase[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] --camera GPC1
     319           xccal.pl --dbname test_xcam --xccal_id $cal_ids[$n] --xcstack_id $stk_ids[$n] --path_base $pathbase[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] --camera GPC1
     320    else
     321      xcskytool -dbname test_xcam -addxccalresult -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].cal.$cal_ids[$n] -hostname elepaio -zpt_obs 25.0 -xccal_id $cal_ids[$n]
     322      xcskytool -dbname test_xcam -xccalresult -xccal_id $cal_ids[$n]
     323    endif
    301324    @ n++
    302325  end
    … …  
    315338
    316339foreach label (0 1 2)
    317   xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam
    318   xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir here/xcff -xcamera test_xcam
     340  xcfftool -dbname test_cam1 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir $mydir/xcff -xcamera test_xcam
     341  xcfftool -dbname test_cam2 -definebyquery -select_xccal_label Test.xccal.$label -set_label Test.xcff.$label -set_workdir $mydir/xcff -xcamera test_xcam
    319342end
    320343
    … …  
    324347  set cam = cam1
    325348  xcfftool -dbname test_$cam -todo -label Test.xcff.$label > xcff.todo.md
    326   set xcff_ids   = (`grep xcff_id   xcff.todo.md | awk '{print $3}'`)
    327   set warp_ids   = (`grep warp_id   xcff.todo.md | awk '{print $3}'`)
     349  set xcff_ids   = (`grep xcff_id           xcff.todo.md | awk '{print $3}'`)
     350  set warp_ids   = (`grep warp_id           xcff.todo.md | awk '{print $3}'`)
     351  set Tskycells  = (`grep skycell_id        xcff.todo.md | awk '{print $3}'`)
     352  set workdir    = (`grep workdir           xcff.todo.md | awk '{print $3}'`)
     353  set warppaths  = (`grep warp_path_base    xcff.todo.md | awk '{print $3}'`)
     354  set srcpaths   = (`grep sources_path_base xcff.todo.md | awk '{print $3}'`)
    328355 
    329356  ## pretend to run xcsky.pl
    330357  set n = 1
    331358  while ($n <= $#xcff_ids)
    332     xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0
     359    # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.wrp.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID ***** TBD *****
     360    if (1 && ($cam == "cam1") && ($n == 1)) then
     361      mkdir -p $workdir[$n]/Fake.Tess/$Tskycells[$n]
     362      echo xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1
     363           xcff_warp.pl --dbname test_$cam --xcff_id $xcff_ids[$n] --warp_id $warp_ids[$n] --skycell_id $Tskycells[$n] --warp_path_base $warppaths[$n] --sources_path_base $srcpaths[$n] --outroot $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] --camera GPC1
     364    else
     365    xcfftool -dbname test_$cam -addresult -xcff_id $xcff_ids[$n] -warp_id $warp_ids[$n] -path_base $workdir[$n]/Fake.Tess/$Tskycells[$n]/Fake.Tess.$Tskycells[$n].wrp.$warp_ids[$n].xcff.$xcff_ids[$n] -hostname elepaio -quality 0 -fault 0
    333366    xcfftool -dbname test_$cam -result    -xcff_id $xcff_ids[$n]
    334367    @ n++
    … …  
    336369
    337370  # get a list of the unique xcff_ids
    338   set xcff_ids   = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`)
     371  set xcff_ids_u   = (`grep xcff_id xcff.todo.md | awk '{print $3}' | sort | uniq`)
    339372
    340373  ## pretend to run xcsky.pl
    341374  set n = 1
    342   while ($n <= $#xcff_ids)
    343     xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids[$n]
    344     xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids[$n] -path_base path/Fake.Tess.xcff.$xcff_ids[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio
    345     xcfftool -dbname test_$cam -summary    -xcff_id $xcff_ids[$n]
     375  while ($n <= $#xcff_ids_u)
     376    # get the workdir from above which matches this uniq xcff_ids:
     377    set workdir_u = ""
     378    set skycell_u = ""
     379    set t = 1
     380    while ($t <= $#xcff_ids)
     381      if ($xcff_ids[$t] == $xcff_ids_u[$n]) then
     382        set workdir_u = $workdir[$n]
     383        set skycell_u = $Tskycells[$n]
     384      endif
     385      @ t++
     386    end
     387    if ($workdir_u == "") then
     388      echo "ERROR: failed to find workdir for $xcff_ids_u[$n] from $workdir"
     389      exit 2
     390    endif
     391
     392    # OUTROOT is defined in xcff.pro based on workdir : "%s/%s/%s/%s.%s.xcff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $XCFF_ID **** TBD ***
     393    xcfftool -dbname test_$cam -toadvance -xcff_id $xcff_ids_u[$n]
     394    xcfftool -dbname test_$cam -addsummary -xcff_id $xcff_ids_u[$n] -path_base $workdir_u/Fake.Tess/$skycell_u/Fake.Tess.$skycell_u.xcff.$xcff_ids_u[$n] -dtime_script 0.0 -quality 0 -fault 0 -hostname elepaio
     395    xcfftool -dbname test_$cam -summary    -xcff_id $xcff_ids_u[$n]
    346396    @ n++
    347397  end
Note: See TracChangeset for help on using the changeset viewer.