Changes between Version 62 and Version 63 of STS.refstack.20120510

Timestamp:

Jun 21, 2012, 11:01:00 AM (14 years ago)

Author:

Mark Huber

Comment:

--

STS.refstack.20120510

@@ -783 +783 @@
 Sample diffims for projection 1 faulted in publishing. So this is more of an example of some things of what not to do..
 
---> dont know how to tell fwhm other than from db - do 1.03 since edge field and wacky overlap, but need to know what chips go into that? no, camProcessedExp
- * need to know rough FWHM of refstack and nightly exposures to pick sample
+-->   -- 
+Nights to pick for test -- do test run skycell.1.03 again since edge field and can have overlap.  Not clear way to get FWHM other than from gpc1 DB, need to know what chips go into that? No, camProcessedExp sufficient.
+ * need to know rough FWHM of refstack and nightly exposures to pick sample -- rough FWHM for refstack in log of run, nightly exposures in DB
+  * refstack ~3.94
+{{{
+SELECT exp_id,exp_time,dateobs,zpt_obs,zpt_stdev,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(iq_fwhm_major/iq_fwhm_minor,3) as iq_elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id)  WHERE  comment like "STS%PP1%" group by exp_id;
+}}}
+  * 120420 - look ok
+  * 120507 ~7.52 (7.9 in cam, wrong way even for cam/warp scaling) -- looked poor, don't use as test
  * need to pick one from 2011 not used in refstack 
-
-SELECT exp_id,exp_time,dateobs,zpt_obs,zpt_stdev,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(iq_fwhm_major/iq_fwhm_minor,3) as iq_elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id)  WHERE  comment like "STS%PP1%" group by exp_id;
-
- * refstack ~3.94
- * 120420 - look ok
- * 120507 ~7.52 (7.9 in cam, wrong way even for cam/warp scaling) -- looked poor, don't use as test
-
-
-select exp_id,exp_time,dateobs,zpt_obs,zpt_stdev,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(iq_fwhm_major/iq_fwhm_minor,3) as iq_elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id) join fakeRun using (cam_id) join warpRun using (fake_id) join stackInputSkyfile using (warp_id) join stackRun using (stack_id) WHERE stackRun.data_group like "STS.refstack.20120510" and stackRun.skycell_id like "%1.03" and comment like "%PP1%" group by exp_id;
-
+{{{
 SELECT exp_id,exp_time,dateobs,zpt_obs,zpt_stdev,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(iq_fwhm_major/iq_fwhm_minor,3) as iq_elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id) join fakeRun using (cam_id) join warpRun using (fake_id) WHERE warpRun.data_group like "STS.refstack.20120510" and comment like "%PP1%" group by exp_id;
 
-
-//SELECT exp_id,exp_time,dateobs,zpt_obs,zpt_stdev,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(iq_fwhm_major/iq_fwhm_minor,3) as iq_elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id) join fakeRun using (cam_id) join warpRun using (fake_id) WHERE warpRun.data_group like "STS.refstack.20120510" and comment like "%PP1%" and not in ( select * FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id) join fakeRun using (cam_id) join warpRun using (fake_id) join stackInputSkyfile using (warp_id) join stackRun using (stack_id) WHERE stackRun.data_group like "STS.refstack.20120510" and stackRun.skycell_id like "%1.03" and comment like "%PP1%" ) group by exp_id;
-
-
 | 393855 | o5822g0346o |       20 | 2011-09-18 09:25:14 | 24.6354 |    3.87493 | 1.166 | 1.08        | 105.39   | 13.63    | 4.11  | 3.61  | 8444848 | STS.000.GIQ PP1 dither 4 | 
-
-
---> 2012 sample to test before updating 
+}}}
+
+2012 sample to test before updating 
+{{{
 SELECT exp_id,exp_name,exp_time,dateobs,zpt_obs,fwhm_major,format(fwhm_major/fwhm_minor,3) as elong,format(fwhm_major-iq_fwhm_major,2) as fwhm_iq_maj,format(camProcessedExp.bg,2) as bg,format(camProcessedExp.bg_stdev,2) as bg_stdev,format(fwhm_major_uq,2) as uq,format(fwhm_major_lq,2) as lq,n_stars,comment FROM rawExp join chipRun using (exp_id) join camRun using (chip_id) join camProcessedExp using (cam_id)  WHERE  comment like "STS%PP1%" group by exp_id;
 
 | 475121 | o6029g0441o |       30 | 2012-04-12 13:40:09 | 24.5321 |    4.06258 | 1.119 | 1.05        | 315.49   | 21.55    | 4.21  | 3.82  | 8078200 | STS.180.ND PP1 center    | 
-
+...
 | 476338 | o6031g0408o |       20 | 2012-04-14 13:34:26 | 24.5537 |    4.27532 | 1.180 | 0.98        | 117.51   | 14.15    | 4.77  | 4.05  | 7485604 | STS.180.GIQ PP1 dither 1 | 
-
+...
 | 477549 | o6033g0430o |       30 | 2012-04-16 13:31:05 | 24.5586 |    7.96196 | 1.194 | 1.83        | 154.22   | 18.30    | 8.08  | 7.71  | 4586012 | STS.180.ND PP1 center    | 
-
-++ | 478479 | o6037g0401o |       20 | 2012-04-20 13:46:26 | 24.5799 |     4.1334 | 1.174 | 1.06        | 76.31    | 12.57    | 4.42  | 3.87  | 8291412 | STS.180.GIQ PP1 dither 1 | 
-
+...
+++++ | 478479 | o6037g0401o |       20 | 2012-04-20 13:46:26 | 24.5799 |     4.1334 | 1.174 | 1.06        | 76.31    | 12.57    | 4.42  | 3.87  | 8291412 | STS.180.GIQ PP1 dither 1 | 
+...
 | 479810 | o6040g0441o |       30 | 2012-04-23 13:31:03 |  24.608 |    6.12941 | 1.131 | 1.20        | 147.86   | 17.88    | 6.39  | 5.88  | 5637788 | STS.180.ND PP1 center    | 
-
+...
 | 482593 | o6045g0420o |       30 | 2012-04-28 13:37:39 | 24.6132 |    6.26513 | 1.051 | 1.07        | 109.03   | 16.37    | 6.59  | 6.09  | 5474888 | STS.180.ND PP1 center    | 
-
+...
 | 483576 | o6049g0286o |       30 | 2012-05-02 13:09:26 | 24.5922 |    4.77424 | 1.088 | 0.94        | 166.69   | 17.89    | 5.10  | 4.62  | 7102152 | STS.180.ND PP1 center    | 
-
-/// | 484578 | o6054g0305o |       30 | 2012-05-07 12:48:56 | 24.5422 |    7.94199 | 1.042 | 0.76        | 297.28   | 23.85    | 8.11  | 7.69  | 3526740 | STS.180.ND PP1 center    | 
-
+...
+xxxx  | 484578 | o6054g0305o |       30 | 2012-05-07 12:48:56 | 24.5422 |    7.94199 | 1.042 | 0.76        | 297.28   | 23.85    | 8.11  | 7.69  | 3526740 | STS.180.ND PP1 center    | 
+...
 | 487890 | o6062g0361o |       30 | 2012-05-15 12:00:59 | 24.5415 |    4.14248 | 1.177 | 1.04        | 146.86   | 16.61    | 4.58  | 3.86  | 8390052 | STS.180.ND PP1 center    | 
+}}}
+
 --> on disk still so try also 
 --> need warp_ids? try exposure name - in any case need to see state in - yes exp_name works - 
+
 select * from warpRun join fakeRun using (fake_id) join camRun using (cam_id) join chipRun using (chip_id) join rawExp using (exp_id) where exp_name="o6062g0361o";
 
@@ -833 +831 @@
 
 
+{{{
 select uri from warpSkyfile where warp_id=409660 and skycell_id like "%1.03";
+
 ./run_ppsub_night_ws.csh gpc1/STS.refstack.20120510/o5822g0346o.393855/o5822g0346o.393855.wrp.409660.skycell.1.03.fits  gpc1/STS.reftest.20120510/STS.V4/skycell.1.03/STS.V4.skycell.1.03.stk.899918.fits 20110918c > runppsub_sts103_110918_1c.bat
 source runppsub_sts103_110918_1c.bat >& runppsub_sts103_110918_1c.log &
 --> good - 
-
+}}}
+
+{{{
 select warpSkyfile.uri from warpSkyfile join warpRun using (warp_id) join fakeRun using (fake_id) join camRun using (cam_id) join chipRun using (chip_id) join rawExp using (exp_id) where exp_name="o6029g0441o" and warpSkyfile.skycell_id like "%1.03";
+
 ./run_ppsub_night_ws.csh  gpc1/STS.nt/2012/04/12//o6029g0441o.475121/o6029g0441o.475121.wrp.396474.skycell.1.03.fits  gpc1/STS.reftest.20120510/STS.V4/skycell.1.03/STS.V4.skycell.1.03.stk.899918.fits 20120412c > runppsub_sts103_120412_1c.bat
 source runppsub_sts103_120412_1c.bat >& runppsub_sts103_120412_1c.log &
 --> bilster cores (undersubs)
-
+}}}
+
+{{{
 select warpSkyfile.uri from warpSkyfile join warpRun using (warp_id) join fakeRun using (fake_id) join camRun using (cam_id) join chipRun using (chip_id) join rawExp using (exp_id) where warpSkyfile.skycell_id like "%1.03" and exp_name="o6031g0408o";
+
 ./run_ppsub_night_ws.csh gpc1/STS.nt/2012/04/14//o6031g0408o.476338/o6031g0408o.476338.wrp.397423.skycell.1.03.fits   gpc1/STS.reftest.20120510/STS.V4/skycell.1.03/STS.V4.skycell.1.03.stk.899918.fits 20120414c > runppsub_sts103_120414_1c.bat
 source runppsub_sts103_120414_1c.bat >& runppsub_sts103_120414_1c.log &
 --> blister cores (undersub)
-
+}}}
+
+{{{
 ./run_ppsub_night_ws.csh gpc1/STS.nt/2012/04/16//o6033g0430o.477549/o6033g0430o.477549.wrp.398532.skycell.1.03.fits  gpc1/STS.reftest.20120510/STS.V4/skycell.1.03/STS.V4.skycell.1.03.stk.899918.fits 20120416c > runppsub_sts103_120416_1c.bat
 source runppsub_sts103_120416_1c.bat >& runppsub_sts103_120416_1c.log &
 --> large conv, oversub
+}}}
 
 ./run_ppsub_night_ws.csh gpc1/STS.nt/2012/04/23//o6040g0441o.479810/o6040g0441o.479810.wrp.401931.skycell.1.03.fits   gpc1/STS.reftest.20120510/STS.V4/skycell.1.03/STS.V4.skycell.1.03.stk.899918.fits 20120423c > runppsub_sts103_120423_1c.bat
@@ -867 +876 @@
 source runppsub_sts103_120515_1c.bat >& runppsub_sts103_120515_1c.log &
 --> low conv, blister undersubs
+}}}
 
 --> looks like 110918, 120420, 120423/120502 -- use 4/23, seeing some odd conv/pixrej artifacts coming out in diffim but not sure if detected and 30s version
-
-
-
 --> may also want to try unconv since STS is supposed to be done w/in certain IQ?
 
 
--- set select nights up for updates - number exposures ~ 16(20s)/16(20s)/12(30s) for 110918, 120420, 120423
---> label STS.mehdifftest
+ * set select nights up for updates - number exposures ~ 16(20s)/16(20s)/12(30s) for 20110918, 20120420, 20120423
+  * label STS.mehdifftest
 --> will need to script mysql call based on what to get the chip/warp_id? detrend different since? may not be unique run given reprocessing for refstack oh well.. need PP1 as well
 --> manually checked and all unique..
-
+{{{
 select exp_name,dateobs,chip_id,warp_id from warpRun join fakeRun using (fake_id) join camRun using (cam_id) join chipRun using (chip_id) join rawExp using (exp_id) where comment like "STS%PP1%" and dateobs like "2011-09-18%" group by exp_name;
 
 chiptool -dbname gpc1 -setimfiletoupdate -set_label STS.mopsdiff.mehtest -chip_id     
 warptool -dbname gpc1  -setskyfiletoupdate -set_label STS.mopsdiff.mehtest -warp_id  
+}}}
 
 --> will have some clean state issues possibly.. 
 --> note dateobs is 2 groups..
-*** 110918 may have magic issues.. may have to just reprocess it - try update since just a test anyways and see -- did update for test case anyways so should work
+
+ * 20110918 during magicked era, keep eye out for any issues. may have to just reprocess it - try update since just a test anyways and see -- did update for test case anyways so should work
 
 mysql -hippdb01 -uippuser -pippuser gpc1 -B -e 'select distinct exp_name,dateobs,chip_id,warp_id from warpRun join fakeRun using (fake_id) join camRun using (cam_id) join chipRun using (chip_id) join rawExp using (exp_id) where comment like "STS%PP1%" and dateobs like "2011-09-18%" and warpRun.state like "cleaned" and chipRun.state like "cleaned" group by exp_name; ' | grep -v chip_id | awk '{print "chiptool -dbname gpc1 -setimfiletoupdate -set_label STS.mopsdiff.mehtest -chip_id "$4"; warptool -dbname gpc1  -setskyfiletoupdate -set_label STS.mopsdiff.mehtest -warp_id  "$5}' > ~/updatechipwarp_sts.mopsdiff.mehtest_110918.bat
@@ -910 +919 @@
 
 
--- setup diff run 
+Setup full projection 1 diffim run:
+ 
 label STS.mopsdiff.mehtest
 data_group STS.mopsdiff.mehtest
@@ -935 +945 @@
 
 difftool -dbname gpc1 -definewarpstack -good_frac 0.2 -warp_label STS.mopsdiff.mehtest -stack_label STS.refstack.20120510 -set_dist_group NULL -set_label STS.mopsdiff.mehtest -set_workdir neb://@HOST@.0/meh/STS.mopsdiff.mehtest -set_data_group STS.mopsdiff.mehtest -set_reduction WARPSTACK -bothways -simple -rerun
+
+'''trouble -- left -bothways in run and don't want to do that'''
 
 --> odd fault 4, won't fix -- something in psphot
@@ -963 +975 @@
 | publishRun              | 
 
+{{{
 mysql> select * from publishClient;
 +-----------+--------+-----------------+----------+----------+---------------------------------+---------------------------------------------------+-----------------+---------------+
@@ -976 +989 @@
 |        12 |      1 | IPP-MOPS-TEST-2 | diff     |        0 | neb://any/gpc1/IPP-MOPS-TEST-2/ | IPP detections for MOPS / Output format version 2 | IPP-MOPS-TEST-2 |             2 | 
 |        13 |      1 | IPP-MOPS-LAP    | diff     |        1 | neb://any/gpc1/IPP-MOPS-LAP     | IPP detections from Large Area Processing         | NULL            |             2 | 
+}}}
 
 -definerun:
@@ -1020 +1034 @@
 
 
---> actual fault from log -- could this astrom issue be because of STS?
+--> actual fault from log -- could this astrom issue be because STS fields or ?
+{{{
 neb://any/gpc1/IPP-MOPS-TEST-2//IPP-MOPS-TEST-2.422401.log
 
@@ -1028 +1043 @@
 Running [/home/panstarrs/ipp/psconfig/ipp-20120404.lin64/bin/pubtool -add -pub_id 422401 -path_base neb://any/gpc1/IPP-MOPS-TEST-2//IPP-MOPS-TEST-2.422401 -dtime_script 2.99998372793198 -hostname ippc10 -fault 2 -dbname gpc1]...
 
-astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2)
-
-
+astrom is sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2)
+}}}
+
+{{{
 neb://ipp007.0/meh/STS.mopsdiff.mehtest/STS.V4/skycell.1.02/STS.V4.skycell.1.02.WS.dif.242223
 meh/STS.mopsdiff.mehtest/STS.V4/skycell.1.02/STS.V4.skycell.1.02.WS.dif.242223.cmf
@@ -1048 +1064 @@
 
 
---> attempt revert - 
+ * attempt to revert - 
 pubtool -dbname gpc1 -revert -fault 2 -pub_id 422401
 
@@ -1054 +1070 @@
 --> does it still need to be added to distribution so that it ends up on the datastore?? no looks like in the log that pubtool is run
 
---> publish cleanup/drop - 
+ * publish cleanup/drop - 
+{{{
 pubtool -dbname gpc1 -updaterun -set_state drop -label STS.mopsdiff.mehtest -client_id 1
 pubtool -dbname gpc1 -updaterun -set_state drop -label STS.mopsdiff.mehtest -client_id 5
@@ -1061 +1078 @@
 select pub_id,diff_id,diffRun.data_group from publishRun join publishDone using (pub_id) join diffRun  where publishRun.label like "STS.mopsdiff.mehtest" and publishRun.state="new" and diff_id=stage_id;
 --> missing 110918 oddly, they've ended up in another client_id.. ugh..
-
-
-*** dont want bothways since it is a WS? is there a way to skip those? no, publish runs directly off diffRun settings, need to clean and rerun..
+}}}
+
+
+ * yes, don't want bothways since it is a WS -- is there a way to skip those? no, publish runs directly off diffRun settings, need to clean and rerun..
 --> this true of other WS runs? OSS is also not bothways, MD same.. so need to rerun and set previous pub to drop and clean diffRun
 
@@ -1150 +1168 @@
 
 
-
 == Diffim sample 2 (published) ==
 Rerun of sample diffims for projection 1 and published under IPP-MOPS-TEST-2