IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changes between Version 21 and Version 22 of PS1_IPP_Czarlog_20190524


Ignore:
Timestamp:
May 30, 2019, 12:34:05 AM (7 years ago)
Author:
cclin33
Comment:

--

Legend:

Unmodified
Added
Removed
Modified

  • PS1_IPP_Czarlog_20190524

    v21 v22  
    410410}}}
    411411
     412 '''update progress'''
     413 The latest results from TdB's mysql code,
     414{{{
     415mysql -hscidbm -uX -pX gpc1 -B -e 'SELECT count(*),gather.data_group,gather.label FROM (SELECT suba.exp_id,subb.warp_id,subb.chip_id,subb.state,subb.data_group,subb.label FROM (SELECT exp_id FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE warpRun.state NOT LIKE "%full%" AND rawExp.camera LIKE "%GPC1%" AND camRun.label LIKE "%LAP.PV3%" GROUP BY exp_id) as suba INNER JOIN (SELECT * FROM (SELECT warp_id,chipRun.exp_id,chipRun.chip_id,warpRun.state,warpRun.data_group,camRun.label FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE warpRun.label NOT LIKE "%broke%" AND warpRun.label NOT LIKE "%hold%" AND rawExp.camera LIKE "%GPC1%" ORDER BY warp_id DESC) as od GROUP BY od.exp_id) AS subb ON (subb.exp_id = suba.exp_id)) as gather WHERE (state LIKE "%cleaned" OR state LIKE "update") GROUP BY data_group;'
     416
     417count(*)        data_group      label
     4181       LAP.PV3.20140730.20140914       LAP.PV3.20140730.final
     4191       LAP.PV3.20140730.20141031       LAP.PV3.20140730.final
     4201       LAP.PV3.20140730.20141102       LAP.PV3.20140730.final
     4211       LAP.PV3.20140730.20141108       LAP.PV3.20140730.final
     4223       LAP.PV3.20140730.20150125       LAP.PV3.20140730.dupe
     42324      LAP.PV3.20140730.20150213       LAP.PV3.20140730.final
     4248       LAP.PV3.20140730.20150216       LAP.PV3.20140730.final
     4258       LAP.PV3.20140730.20150218       LAP.PV3.20140730.final
     42642      LAP.PV3.20140730.20150221       LAP.PV3.20140730.final
     427117     LAP.PV3.20140730.20150222       LAP.PV3.20140730.final
     42832      LAP.PV3.20140730.20150223       LAP.PV3.20140730.final
     4291       LAP.PV3.20140730.20150224       LAP.PV3.20140730.final
     4302       LAP.PV3.20140730.bigmem.20150102        LAP.PV3.20140730.final
     4311       LAP.PV3.20140730.bigmem.20150105        LAP.PV3.20140730.final
     4321       LAP.PV3.20140730.bigmem.20150108        LAP.PV3.20140730.final
     433}}}
     434 Above has been update, but some of the exposure chip/warp are keeping on update state, not gone through full state.
     435{{{
     436mysql -hscidbm -ux -px gpc1 -B -e 'SELECT suba.exp_id,subb.warp_id,subb.chip_id,subb.maxstate,subb.data_group,subb.label,subb.maxchipstate FROM (SELECT exp_id FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE warpRun.state NOT LIKE "%full%" AND rawExp.camera LIKE "%GPC1%" AND camRun.data_group LIKE "LAP.PV3.20140730.bigmem.xxxxx" GROUP BY exp_id) as suba INNER JOIN (SELECT * FROM (SELECT warp_id,chipRun.exp_id,chipRun.chip_id,warpRun.state as maxstate,warpRun.data_group,camRun.label,chipRun.state as maxchipstate FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.camera LIKE "%GPC1%" ORDER BY warp_id DESC) as od GROUP BY od.exp_id) AS subb ON (subb.exp_id = suba.exp_id) WHERE subb.maxstate LIKE "cleaned" limit 1000;' | grep -v exp_id | awk '{print "chiptool -dbname gpc1 -updaterun  -set_label update.LAP.PV3 -chip_id  "$3" ; chiptool -dbname gpc1 -setimfiletoupdate -set_label update.LAP.PV3 -chip_id  "$3" ; warptool -dbname gpc1 -updaterun -set_label update.LAP.PV3 -warp_id "$2"; warptool -dbname gpc1  -setskyfiletoupdate -set_label update.LAP.PV3 -warp_id "$2 }' | tcsh
     437
     438447     LAP.PV3.20140730.bigmem.20150112        LAP.PV3.20140730.final 
     439576     LAP.PV3.20140730.bigmem.20150113        LAP.PV3.20140730.final
     440961     LAP.PV3.20140730.bigmem.20150114        LAP.PV3.20140730.final
     441581     LAP.PV3.20140730.bigmem.20150115        LAP.PV3.20140730.final
     442487     LAP.PV3.20140730.bigmem.20150116        LAP.PV3.20140730.final
     44328      LAP.PV3.20140730.bigmem.20150117        LAP.PV3.20140730.final
     444753     LAP.PV3.20140730.bigmem.20150118        LAP.PV3.20140730.final
     4451519    LAP.PV3.20140730.bigmem.20150119        LAP.PV3.20140730.final
     446592     LAP.PV3.20140730.bigmem.20150120        LAP.PV3.20140730.final
     44778      LAP.PV3.20140730.bigmem.20150121        LAP.PV3.20140730.final
     448576     LAP.PV3.20140730.bigmem.20150122        LAP.PV3.20140730.final
     449580     LAP.PV3.20140730.bigmem.20150123        LAP.PV3.20140730.final
     450250     LAP.PV3.20140730.bigmem.20150124        LAP.PV3.20140730.final
     4514884    LAP.PV3.20140730.bigmem.20150125        LAP.PV3.20140730.final
     452310     LAP.PV3.20140730.bigmem.20150128        LAP.PV3.20140730.final
     4532286    LAP.PV3.20140730.bigmem.20150129        LAP.PV3.20140730.final
     454373     LAP.PV3.20140730.bigmem.20150130        LAP.PV3.20140730.final
     455231     LAP.PV3.20140730.bigmem.20150131        LAP.PV3.20140730.final
     4561997    LAP.PV3.20140730.bigmem.20150202        LAP.PV3.20140730.final
     45762      LAP.PV3.20140730.bigmem.20150204        LAP.PV3.20140730.final
     4581228    LAP.PV3.20140730.bigmem.20150205        LAP.PV3.20140730.final
     45968      LAP.PV3.20140730.bigmem.20150206        LAP.PV3.20140730.final
     4602       LAP.PV3.20140730.ipp.20141106   LAP.PV3.20140730.final
     4617       LAP.PV3.20140730.ipp.20141107   LAP.PV3.20140730.final
     4624       LAP.PV3.20140730.ipp.20141108   LAP.PV3.20140730.final
     4632       LAP.PV3.20140730.ipp.20141117   LAP.PV3.20140730.dupe
     4641       LAP.PV3.20140730.ipp.20141118   LAP.PV3.20140730.final
     4653       LAP.PV3.20140730.ipp.20141120   LAP.PV3.20140730.dupe
     4663       LAP.PV3.20140730.ipp.20141121   LAP.PV3.20140730.final
     4671       LAP.PV3.20140730.ipp.20141122   LAP.PV3.20140730.final
     4681       LAP.PV3.20140730.ipp.20150218   LAP.PV3.20140730.final
     4691       LAP.PV3W.20150312.20150320      LAP.PV3W.20150312
     47019      LAP.PV3W.20150312.20150329      LAP.PV3W.20150312
     47116      LAP.PV3W.20150312.20150409      LAP.PV3W.20150312
     4728       LAP.PV3W.20150312.20150413      LAP.PV3W.20150312
     47330      LAP.ThreePi.20120706.20121002   PV2.cleanup
     4742       LAP.ThreePi.20120706.20121004   PV2.cleanup
     4753       LAP.ThreePi.20120706.20121205   PV2.cleanup
     4761       mops.fixbrokenLAP.20180323      ws_nightly_update
     4771       OSS.20140423    OSS.nightlyscience
     47816      ThreePi.20120927        PV2.cleanup
     479}}}
     480
     481 '''cleanup progress'''
     482 Actually, when I check their data_state from chipProcessedImfile table, most of them have been marked as "cleaned".  I also check if there are other states not like 'cleaned' and then send them "goto_cleaned" state.
     483{{{
     484--> to check if there are any non-cleaned data_state.
     485SELECT chipProcessedImfile.data_state,chipProcessedImfile.chip_id,chipRun.chip_id,chipProcessedImfile.quality,chipProcessedImfile.fault FROM chipProcessedImfile, chipRun \
     486WHERE chipProcessedImfile.data_state not like 'cleaned' and chipRun.state LIKE 'wait' AND chipRun.data_group LIKE 'MD07.20%' AND chipRun.chip_id = chipProcessedImfile.chip_id GROUP BY chipProcessedImfile.chip_id,chipProcessedImfile.data_state;
     487
     488+------------+---------+---------+---------+-------+
     489| data_state | chip_id | chip_id | quality | fault |
     490+------------+---------+---------+---------+-------+
     491| purged     |   19573 |   19573 |       0 |     0 |
     492| purged     |   19574 |   19574 |       0 |     0 |
     493| purged     |   19575 |   19575 |       0 |     0 |
     494| purged     |   19576 |   19576 |       0 |     0 |
     495...
     496| full       |  827266 |  827266 |       0 |     0 |
     497| full       |  848347 |  848347 |       0 |     0 |
     498+------------+---------+---------+---------+-------+
     499299 rows in set (0.27 sec)
     500
     501--> this script send "wait" state, but "non-cleaned" date_state to be "goto_cleaned"
     502mysql -hscidbm -uippuser -pippuser gpc1 -B -e '\
     503SELECT chipProcessedImfile.chip_id,chipProcessedImfile.data_state,chipRun.chip_id FROM chipProcessedImfile, chipRun \
     504WHERE chipProcessedImfile.data_state not like "cleaned" \
     505  AND chipRun.state LIKE "wait" \
     506  AND chipRun.data_group LIKE "MD07.20%" \
     507  AND chipRun.chip_id = chipProcessedImfile.chip_id GROUP BY chipProcessedImfile.chip_id,chipProcessedImfile.data_state;' |grep -v data_state | \
     508awk '{print "chiptool -dbname gpc1 -updaterun -set_state goto_cleaned -set_label goto_cleaned -chip_id  "$1}'  |tcsh
     509
     510--> most of "wait" states are in fact with date_state "cleaned", I double check some of them with "neb-stat" and can't find them anymore.
     511SELECT chipProcessedImfile.data_state,chipProcessedImfile.chip_id,chipRun.chip_id,chipProcessedImfile.quality,chipProcessedImfile.fault FROM chipProcessedImfile, chipRun \
     512WHERE chipRun.state LIKE 'wait' AND chipRun.data_group LIKE 'LAP.ThreePi.20120706.2012%' AND chipRun.chip_id = chipProcessedImfile.chip_id GROUP BY chipProcessedImfile.chip_id,chipProcessedImfile.data_state;
     513
     514+------------+---------+---------+---------+-------+
     515| data_state | chip_id | chip_id | quality | fault |
     516+------------+---------+---------+---------+-------+
     517| cleaned    |  657694 |  657694 |       0 |     0 |
     518| cleaned    |  657695 |  657695 |       0 |     0 |
     519| cleaned    |  657696 |  657696 |       0 |     0 |
     520....
     521| cleaned    |  723556 |  723556 |       0 |     0 |
     522| cleaned    |  723557 |  723557 |       0 |     0 |
     523+------------+---------+---------+---------+-------+
     52418378 rows in set (1.75 sec)
     525
     526--> this script send "wait" state, but in fact "cleaned" date_state to be "cleaned" state, which avoid executing cleanup script again.
     527mysql -hscidbm -uippuser -pippuser gpc1 -B -e '\
     528SELECT chip_id,data_group FROM chipRun \
     529WHERE  state like "wait" AND data_group LIKE "MD07.20%";' | grep -v data_group | \
     530awk '{print "chiptool -dbname gpc1 -updaterun -set_state cleaned -set_label goto_cleaned -chip_id  "$1}' | tcsh
     531}}}
     532
     533 From above method, I've checked all "LAP" related chip with "wait" state and set them cleaned.  Do we want to set all of the "wait" state to "cleaned" or just "LAP"?
     534{{{
     535mysql -hscidbm -ux -px gpc1 -B -e 'SELECT waitct,data_group,label from (SELECT count(*) as waitct,data_group,label FROM chipRun WHERE state like "wait"  GROUP BY data_group) as s ORDER by waitct ASC;' | grep LAP
     536}}}
     537
     538
    412539=== Thursday : 2019.05.30 ===