IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 30, 2019, 8:48:56 AM (7 years ago)
Author:
eugene
Message:

merge changes from trunk (night_report tweaks)

File:
1 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-ps2-20190404/ippScripts/scripts/night_report.pl

    r40762 r40766  
    6868}
    6969
    70 ##################33
     70##################
    7171### testing the diff grouping -- this does not work for the Euclid data taken 2018.07.29 because the OBJECT
    7272### field is missing
    … …  
    382382    $query .= "     AND exp_type = 'OBJECT'";
    383383    $query .= "     AND obs_mode NOT LIKE 'ENGINEERING'";
    384     $query .= "     AND obs_mode like '%SS%'";
     384    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    385385    $query .= "     AND rawExp.comment like '%visit%'";
    386386    $query .= "     AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
    … …  
    438438    $query .= "     AND exp_type = 'OBJECT'";
    439439    $query .= "     AND obs_mode NOT LIKE 'ENGINEERING'";
    440     $query .= "     AND obs_mode like '%SS%'";
     440    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    441441    $query .= "     AND rawExp.comment like '%visit%'";
    442442    $query .= "     AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
    … …  
    474474    $query .= "     AND exp_type = 'OBJECT'";
    475475    $query .= "     AND obs_mode NOT LIKE 'ENGINEERING'";
    476     $query .= "     AND obs_mode like '%SS%'";
     476    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    477477    $query .= "     AND rawExp.comment like '%visit%'";
    478478    $query .= "     AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
    … …  
    660660    $query .= "   AND exp_type = 'OBJECT'";
    661661    $query .= "   AND obs_mode NOT LIKE 'ENGINEERING'";
    662     $query .= "   AND obs_mode NOT like '%SS%'";
     662    $query .= "   AND (obs_mode NOT LIKE '%SS%' AND obs_mode NOT LIKE '%BRIGHT%')";
    663663    $query .= "   AND rawExp.comment like '%visit%'";
    664664    $query .= "   AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
    … …  
    694694    $query .= "    WHERE dateobs like '$date%'";
    695695    $query .= "    AND exp_id > $refExpID"; #REF
    696     $query .= "    AND obs_mode like '%SS%'";
     696    $query .= "    AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    697697    $query .= "    AND (camProcessedExp.quality = 0)";
     698    $query .= "    AND (comment NOT LIKE '%twilight%')";
    698699    $query .= "  ) AS expall";
    699700    $query .= "  LEFT JOIN";
    … …  
    709710    $query .= "    JOIN chipRun using (chip_id)";
    710711    $query .= "    JOIN rawExp using (exp_id)";
    711     $query .= "    WHERE diffRun.reduction like 'SWEETSPOT'";
     712    $query .= "    WHERE stack2 IS NULL";
    712713##  $query .= "    AND exp_id > $refExpID"; -- adding this restriction here slows the query way down1
    713714    $query .= "    AND stack2 IS NULL";
    … …  
    724725    while (@row = $result->fetchrow_array()) {
    725726        if ($doHeader) {
    726             print "Unpublished Solar System Exposures ( obs_mode like %SS% ) with good camera quality :\n";
     727            print "Unpublished Solar System Exposures WW diffs (obs_mode like %SS% or obs_mode like %BRIGHT%) with good camera quality :\n";
    727728            printf "%12s %12s %12s | %-19s | %-16s\n", "exp_name", "warp_id", "diff_id", "dateobs", "comment";
    728729            $doHeader = 0;
    … …  
    751752    $query .= "     AND exp_type = 'OBJECT'";
    752753    $query .= "     AND obs_mode NOT LIKE 'ENGINEERING'";
    753     $query .= "     AND obs_mode like '%SS%'";
     754    $query .= "     AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    754755    $query .= "     AND rawExp.comment like '%visit%'";
    755756    $query .= "     AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
    … …  
    770771    print "Current status of SS and Euclid chunks:\n";
    771772
    772     #This is a long query, but it efficiently gathers all the necessary information
    773     my $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits,groupdiff.Ndiffs FROM";
     773    #First find the number of chunks and their names
     774    my $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits FROM";
    774775    $query .= "   (";
    775     $query .= "     SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(*) AS Nquad_firstvisit";
     776    $query .= "     SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nquad_firstvisit";
    776777    $query .= "     FROM rawExp";
    777778    $query .= "     WHERE dateobs LIKE '$date%'";
    … …  
    783784    $query .= "   (";
    784785    $query .= "     SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM ";
    785     $query .= "       (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(*) AS Nvisits";
     786    $query .= "       (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(distinct(comment)) AS Nvisits";
    786787    $query .= "         FROM rawExp";
    787788    $query .= "         WHERE dateobs LIKE '$date%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    … …  
    793794    $query .= "   ) AS group4";
    794795    $query .= "   on groupchunk.chunk=group4.chunk";
    795     $query .= "   LEFT JOIN";
    796     $query .= "   (";
    797     $query .= "     SELECT substr(expall.comment, 1, position(' ' in expall.comment)) AS chunk,count(*) AS Ndiffs FROM ";
    798     $query .= "       (SELECT exp_name,comment,dateobs";
    799     $query .= "         FROM rawExp ";
    800     $query .= "         WHERE dateobs LIKE '$date%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%') AND obs_mode NOT LIKE 'ENGINEERING'";
    801     $query .= "       ) AS expall ";
    802     $query .= "     LEFT JOIN";
    803     $query .= "     (SELECT * FROM ";
    804     $query .= "       (";
    805     $query .= "         (SELECT exp_name FROM ";
    806     $query .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)";
    807     $query .= "           JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
    808     $query .= "           WHERE rawExp.dateobs LIKE '$date%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    809     $query .= "           AND stack2 IS NULL AND diffRun.diff_id=stage_id";
    810     $query .= "         )  ";
    811     $query .= "         UNION ";
    812     $query .= "         (SELECT exp_name FROM";
    813     $query .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)";
    814     $query .= "           JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
    815     $query .= "           WHERE rawExp.dateobs LIKE '$date%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
    816     $query .= "           AND stack2 IS NULL AND diffRun.diff_id=stage_id";
    817     $query .= "         )";
    818     $query .= "       ) AS diffgather ";
    819     $query .= "       GROUP BY diffgather.exp_name";
    820     $query .= "     ) AS exppub";
    821     $query .= "     ON expall.exp_name=exppub.exp_name";
    822     $query .= "     WHERE exppub.exp_name=expall.exp_name";
    823     $query .= "     GROUP BY chunk ";
    824     $query .= "   ) AS groupdiff";
    825     $query .= "   ON groupchunk.chunk=groupdiff.chunk";
    826 
     796 
    827797    my $result = &mysql_select ($query);
    828798
    829799    printf "%18s | %18s | %18s | %18s |\n", "chunk", "Nquad_firstvisit", "Nquad_fourvisits", "Ndiffs done";
    830800    while (@row = $result->fetchrow_array()) {
    831           if ($row[1] ne $row[2]) {
    832             if ($row[3] ne ($row[1]*4)) {printf "%18s | %18s | %18s | %14s%4s | %18s\n", $row[0], $row[1], $row[2], $row[3], "/?  ", "chunk not finished";}  # chunk    Nquad_firstvisit        Nquad_fourvisits        Ndiffs
    833             if ($row[3] eq ($row[1]*4)) {printf "%18s | %18s | %18s | %14s%4s | %18s\n", $row[0], $row[1], $row[2], $row[3], "/?  ", "chunk done";}          # chunk    Nquad_firstvisit        Nquad_fourvisits        Ndiffs
     801          my $chunkname = $row[0];
     802          my $Nfirstvisits = $row[1];
     803          my $Nfourvisits = $row[2];
     804
     805          my $query2 .= "     SELECT warp_id FROM ";
     806          $query2 .= "           warpRun JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
     807          $query2 .= "           WHERE rawExp.dateobs LIKE '$date%' AND (obs_mode LIKE '%SS%' OR obs_mode LIKE '%BRIGHT%')";
     808          $query2 .= "           AND substr(comment, 1, position(' ' in comment)) LIKE '$chunkname%'";
     809          my $result2 = &mysql_select ($query2);
     810
     811          my $Ndiffs = 0;
     812          while (@row2 = $result2->fetchrow_array()) {
     813              my $query3 .= "     SELECT diff_id FROM ";
     814              $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)";
     815              $query3 .= "           WHERE stack2 IS NULL AND warp_id='$row2[0]' GROUP BY diff_id";
     816              my $result3 = &mysql_select ($query3);
     817              while (@row3 = $result3->fetchrow_array()) {if ($row3[0] !=0) {$Ndiffs ++;}}
     818
     819              my $query3 .= "     SELECT diff_id FROM ";
     820              $query3 .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)";
     821              $query3 .= "           WHERE stack2 IS NULL AND warp_id='$row2[0]' GROUP BY diff_id";
     822              my $result3 = &mysql_select ($query3);
     823              while (@row3 = $result3->fetchrow_array()) {if ($row3[0] !=0) {$Ndiffs ++;}}
    834824          }
    835           if ($row[1] eq $row[2]) {
    836             if ($row[3] ne ($row[1]*4)) {printf "%18s | %18s | %18s | %14s%1s%3s | %18s\n", $row[0], $row[1], $row[2], $row[3],"/",$row[1]*4, "chunk not finished";}  # chunk   Nquad_firstvisit        Nquad_fourvisits        Ndiffs
    837             if ($row[3] eq ($row[1]*4)) {printf "%18s | %18s | %18s | %14s%1s%3s | %18s\n", $row[0], $row[1], $row[2], $row[3],"/",$row[1]*4, "chunk done";}          # chunk   Nquad_firstvisit        Nquad_fourvisits        Ndiffs
     825
     826          if ($Nfirstvisits ne $Nfourvisits) {
     827            if ($Ndiffs ne ($Nfirstvisits*4)) {printf "%18s | %18s | %18s | %14s%4s | %18s\n", $chunkname, $Nfirstvisits, $Nfourvisits, $Ndiffs, "/?  ", "chunk not finished";}  # chunk        Nquad_firstvisit        Nquad_fourvisits        Ndiffs
     828            if ($Ndiffs eq ($Nfirstvisits*4)) {printf "%18s | %18s | %18s | %14s%4s | %18s\n", $chunkname, $Nfirstvisits, $Nfourvisits, $Ndiffs, "/?  ", "chunk done";}          # chunk        Nquad_firstvisit        Nquad_fourvisits        Ndiffs
     829          }
     830          if ($Nfirstvisits eq $Nfourvisits) {
     831            if ($Ndiffs ne ($Nfirstvisits*4)) {printf "%18s | %18s | %18s | %14s%1s%3s | %18s\n", $chunkname, $Nfirstvisits, $Nfourvisits, $Ndiffs,"/",$Nfirstvisits*4, "chunk not finished";}  # chunk Nquad_firstvisit        Nquad_fourvisits        Ndiffs
     832            if ($Ndiffs eq ($Nfirstvisits*4)) {printf "%18s | %18s | %18s | %14s%1s%3s | %18s\n", $chunkname, $Nfirstvisits, $Nfourvisits, $Ndiffs,"/",$Nfirstvisits*4, "chunk done";}          # chunk Nquad_firstvisit        Nquad_fourvisits        Ndiffs
    838833          }
    839834    }
Note: See TracChangeset for help on using the changeset viewer.