IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changes between Version 8 and Version 9 of PS1_IPP_Czarlog_20190503


Ignore:
Timestamp:
May 6, 2019, 12:05:58 PM (7 years ago)
Author:
tdeboer
Comment:

--

Legend:

Unmodified
Added
Removed
Modified

  • PS1_IPP_Czarlog_20190503

    v8 v9  
    1141141759514 full tdbdifftest tdbdifftest.20190503 NULL 0 RINGS.V3 1 2019-05-04 00:17:42 SWEETSPOT neb://@HOST@.0/gpc1/tdbdifftest.20190503         
    115115}}}
     116 Now then, we can simulate losing a warp exposure by changing its label and doing query only for the tdbdifftest label (in warpRun and diffRun in this case). In this case, lose the visit1 warp and the v1-v2 diff:
     117{{{
     118warptool -dbname gpc1 -updaterun -set_label tdbdifftest1 -warp_id 2091717
     119difftool -dbname gpc1 -updaterun -set_label tdbdifftest1 -diff_id 1759513
     120}}}
     121 Then, run a query to get the information:
     122{{{
     123mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND warpRun.label LIKE 'tdbdifftest' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.object;"
     124}}}
     125 which returns:
     126{{{
     127chunk           object          warp1   warp2   warp3   warp4   diff1   diff2   diff3   diff4   workdir label   data_group      reduction
     128162.OSS.O.Q.w   ps1_14_2124     0       2091722 2091718 2091720 0       0       1759514 1759514 neb://@HOST@.0/gpc1/20130319    tdbdifftest     tdbdifftest.20190503    SWEETSPOT
     129}}}
     130 That shows clearly that 1 warp is missing and two visits have been diffed. Running the desperate diffs creation script with the recipe from the PSNC_MOPS wiki page:
     131{{{
     132mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND warpRun.label LIKE 'tdbdifftest' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.object;" | grep -v object | awk '{nwarp=0; if($3!=0) nwarp++; if($4!=0) nwarp++; if($5!=0) nwarp++; if($6!=0) nwarp++ ; ndiff=0; if($7!=0) ndiff++; if($8!=0) ndiff++; if($9!=0) ndiff++; if($10!=0) ndiff++ ; if(nwarp == 3 && nwarp !=ndiff) {if($7==0 && $3==0){w1=$4; w2=$5} else if($8==0 && $4==0){w1=$3; w2=$5} else if($9==0 && $5==0){w1=$4; w2=$6} else if($10==0 && $6==0){w1=$4; w2=$5}; print "difftool -dbname gpc1 -definewarpwarp -warp_id "w1" -template_warp_id "w2" -backwards -set_workdir "$11" -set_dist_group NULL -set_label "$12" -set_data_group "$13" -set_reduction "$14" -simple -rerun -set_note desperate diff  -pretend";}}'
     133}}}
     134 and returns:
     135{{{
     136difftool -dbname gpc1 -definewarpwarp -warp_id 2091722 -template_warp_id 2091718 -backwards -set_workdir neb://@HOST@.0/gpc1/20130319 -set_dist_group NULL -set_label tdbdifftest -set_data_group tdbdifftest.20190503 -set_reduction SWEETSPOT -simple -rerun -set_note desperate diff  -pretend
     137}}}
     138 This is the desired results!
    116139
     140 ----------------------------
     141 Now, we know this is not something that happens in really. The first diffable pair will always be diffed. Simulate that instead by leaving the warpID label as is (first visit missing), but putting both diffRun labels to off. Then, make a v2-v3 diff:
     142{{{
     143difftool -dbname gpc1 -definewarpwarp -warp_id 2091722 -template_warp_id 2091718 -backwards  -set_workdir neb://@HOST@.0/gpc1/tdbdifftest.20190503 -set_dist_group NULL -set_label tdbdifftest -set_data_group tdbdifftest.20190503 -set_reduction SWEETSPOT -simple -rerun
     144}}}
     145 Then, we have the following diffs:
     146{{{
     147diff_ID state label data grp dist grp Magicked tess ID both ways Time registered reduction workdir
     1481759513 full tdbdifftest1 tdbdifftest.20190503 NULL 0 RINGS.V3 1 2019-05-04 00:17:41 SWEETSPOT neb://@HOST@.0/gpc1/tdbdifftest.20190503         
     1491759514 full tdbdifftest1 tdbdifftest.20190503 NULL 0 RINGS.V3 1 2019-05-04 00:17:42 SWEETSPOT neb://@HOST@.0/gpc1/tdbdifftest.20190503         
     1501760928 full tdbdifftest  tdbdifftest.20190503 NULL 0 RINGS.V3 1 2019-05-06 21:16:16 SWEETSPOT neb://@HOST@.0/gpc1/tdbdifftest.20190503         
     151}}}
     152 Now run the query script, which returns:
     153{{{
     154chunk           object          warp1   warp2   warp3   warp4   diff1   diff2   diff3   diff4   workdir label   data_group      reduction
     155162.OSS.O.Q.w   ps1_14_2124     0       2091722 2091718 2091720 0       1760928 1760928 0       neb://@HOST@.0/gpc1/20130319    tdbdifftest     tdbdifftest.20190503    SWEETSPOT
     156}}}
     157 This looks like what we expect. Run the desperate diff creation:
     158{{{
     159mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND warpRun.label LIKE 'tdbdifftest' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.object;" | grep -v object | awk '{nwarp=0; if($3!=0) nwarp++; if($4!=0) nwarp++; if($5!=0) nwarp++; if($6!=0) nwarp++ ; ndiff=0; if($7!=0) ndiff++; if($8!=0) ndiff++; if($9!=0) ndiff++; if($10!=0) ndiff++ ; if(nwarp == 3 && nwarp !=ndiff) {if($7==0 && $3==0){w1=$4; w2=$5} else if($8==0 && $4==0){w1=$3; w2=$5} else if($9==0 && $5==0){w1=$4; w2=$6} else if($10==0 && $6==0){w1=$4; w2=$5}; print "difftool -dbname gpc1 -definewarpwarp -warp_id "w1" -template_warp_id "w2" -backwards -set_workdir "$11" -set_dist_group NULL -set_label "$12" -set_data_group "$13" -set_reduction "$14" -simple -rerun -set_note desperate diff  -pretend";}}'
     160}}}
     161 which returns:
     162{{{
     163difftool -dbname gpc1 -definewarpwarp -warp_id 2091722 -template_warp_id 2091718 -backwards -set_workdir neb://@HOST@.0/gpc1/20130319 -set_dist_group NULL -set_label tdbdifftest -set_data_group tdbdifftest.20190503 -set_reduction SWEETSPOT -simple -rerun -set_note desperate diff  -pretend
     164}}}
     165 So, the 'usual' recipe does not work in all cases. i.e. it works off of a set strategy but does not check against existing (and therefore missing) diffs. For now, we can adjust the fixed recipe based on the fact that we know the first diffable pair will always be diffed. This affects the case of v1 or v2 being missing. v1 missing results in v2-v3 diff being made and v3-v4 being needed; v2 missing results in v1-v3 being made and v3-v4 being needed. Try this instead:
     166{{{
     167mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND warpRun.label LIKE 'tdbdifftest' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND diffRun.label LIKE 'tdbdifftest' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.object;" | grep -v object | awk '{nwarp=0; if($3!=0) nwarp++; if($4!=0) nwarp++; if($5!=0) nwarp++; if($6!=0) nwarp++ ; ndiff=0; if($7!=0) ndiff++; if($8!=0) ndiff++; if($9!=0) ndiff++; if($10!=0) ndiff++ ; if(nwarp == 3 && nwarp !=ndiff) {if($7==0 && $3==0){w1=$5; w2=$6} else if($8==0 && $4==0){w1=$5; w2=$6} else if($9==0 && $5==0){w1=$4; w2=$6} else if($10==0 && $6==0){w1=$4; w2=$5}; print "difftool -dbname gpc1 -definewarpwarp -warp_id "w1" -template_warp_id "w2" -backwards -set_workdir "$11" -set_dist_group NULL -set_label "$12" -set_data_group "$13" -set_reduction "$14" -simple -rerun -set_note desperate diff  -pretend";}}'
     168}}}
     169 which returns:
     170{{{
     171difftool -dbname gpc1 -definewarpwarp -warp_id 2091718 -template_warp_id 2091720 -backwards -set_workdir neb://@HOST@.0/gpc1/20130319 -set_dist_group NULL -set_label tdbdifftest -set_data_group tdbdifftest.20190503 -set_reduction SWEETSPOT -simple -rerun -set_note desperate diff  -pretend
     172}}}
     173 That now works. But clearly, want to make an explicit check against the diff_ids available.
     174
     175 In conclusion, to create the correct desperate diffs for a given chunk, run:
     176{{{
     177mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND substr(comment, 1, position(' ' in comment)) LIKE 'OSSR.R10S1.14.Q.w%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND substr(comment, 1, position(' ' in comment)) LIKE 'OSSR.R10S1.14.Q.w%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND substr(comment, 1, position(' ' in comment)) LIKE 'OSSR.R10S1.14.Q.w%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.object;" | grep -v object | awk '{nwarp=0; if($3!=0) nwarp++; if($4!=0) nwarp++; if($5!=0) nwarp++; if($6!=0) nwarp++ ; ndiff=0; if($7!=0) ndiff++; if($8!=0) ndiff++; if($9!=0) ndiff++; if($10!=0) ndiff++ ; if(nwarp == 3 && nwarp !=ndiff) {if($7==0 && $3==0){w1=$5; w2=$6} else if($8==0 && $4==0){w1=$5; w2=$6} else if($9==0 && $5==0){w1=$4; w2=$6} else if($10==0 && $6==0){w1=$4; w2=$5}; print "difftool -dbname gpc1 -definewarpwarp -warp_id "w1" -template_warp_id "w2" -backwards -set_workdir "$11" -set_dist_group NULL -set_label "$12" -set_data_group "$13" -set_reduction "$14" -simple -rerun -set_note desperate diff  -pretend";}}' | bash
     178}}}
     179 or for a whole night, using:
     180{{{
     181mysql -hX -uX -pippuser gpc1 -B -e "SELECT selchunk.chunk,selchunk.object,MAX(CASE WHEN selchunk.visit=1 THEN selchunk.warp_id ELSE 0 END) as warp1,MAX(CASE WHEN selchunk.visit=2 THEN selchunk.warp_id ELSE 0 END) as warp2,MAX(CASE WHEN selchunk.visit=3 THEN selchunk.warp_id ELSE 0 END) as warp3,MAX(CASE WHEN selchunk.visit=4 THEN selchunk.warp_id ELSE 0 END) as warp4,MAX(CASE WHEN diffchunk.visit=1 THEN diffchunk.diff_id ELSE 0 END) as diff1,MAX(CASE WHEN diffchunk.visit=2 THEN diffchunk.diff_id ELSE 0 END) as diff2,MAX(CASE WHEN diffchunk.visit=3 THEN diffchunk.diff_id ELSE 0 END) as diff3,MAX(CASE WHEN diffchunk.visit=4 THEN diffchunk.diff_id ELSE 0 END) as diff4,selchunk.workdir,selchunk.label,selchunk.data_group,selchunk.reduction FROM (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit,object,warp_id,rawExp.workdir,chipRun.label,chipRun.data_group,rawExp.reduction FROM warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' ORDER BY warp_id DESC) as selchunk LEFT JOIN ((SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp1=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' AND stack2 IS NULL GROUP BY warp_id) UNION (SELECT SUBSTRING_INDEX(comment, ' ',1) AS chunk,SUBSTRING_INDEX(comment, ' ',-1) AS visit, object,warp_id,diff_id FROM diffRun JOIN diffInputSkyfile USING (diff_id) JOIN warpRun ON (warp2=warp_id) JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id) WHERE rawExp.exp_name LIKE 'o%' AND rawExp.dateobs LIKE '`date -u "+%Y-%m-%d"`%' AND stack2 IS NULL GROUP BY warp_id)) as diffchunk ON selchunk.warp_id=diffchunk.warp_id GROUP BY selchunk.chunk,selchunk.object;" | grep -v object | awk '{nwarp=0; if($3!=0) nwarp++; if($4!=0) nwarp++; if($5!=0) nwarp++; if($6!=0) nwarp++ ; ndiff=0; if($7!=0) ndiff++; if($8!=0) ndiff++; if($9!=0) ndiff++; if($10!=0) ndiff++ ; if(nwarp == 3 && nwarp !=ndiff) {if($7==0 && $3==0){w1=$5; w2=$6} else if($8==0 && $4==0){w1=$5; w2=$6} else if($9==0 && $5==0){w1=$4; w2=$6} else if($10==0 && $6==0){w1=$4; w2=$5}; print "difftool -dbname gpc1 -definewarpwarp -warp_id "w1" -template_warp_id "w2" -backwards -set_workdir "$11" -set_dist_group NULL -set_label "$12" -set_data_group "$13" -set_reduction "$14" -simple -rerun -set_note desperate diff  -pretend";}}' | bash
     182}}}
    117183
    118184=== Tuesday : 2019.05.07 ===