Changeset 40666 for trunk

Timestamp:

Apr 3, 2019, 1:34:04 PM (7 years ago)

Author:

tdeboer

Message:

update to night_report.pl to show info on chunks

File:

• trunk/ippScripts/scripts/night_report.pl (modified) (9 diffs)

trunk/ippScripts/scripts/night_report.pl

@@ -16 +16 @@
     'report'            => \$report,   # Formatted for ps-obs report
     'very-verbose'      => \$verbose2, # Print to stdout
+    'chunks'            => \$chunks,   # give information on the current status of the chunks
     ) or pod2usage( 2 );
 
@@ -76 +77 @@
     # my $query = "SELECT Nvisit, count(*) FROM";
     # $query .= " (";
-    # $query .= "   SELECT object, filter, chunk, count(*) as Nvisit FROM";
-    # $query .= "   (";
-    my $query = "SELECT object, filter, comment, 'S', substr(comment, 1, position(' ' in comment)) as chunk, 'E'";
+    my $query = "   SELECT object, filter, chunk, count(*) as Nvisit FROM";
+    $query .= "   (";
+    $query .= "SELECT object, filter, comment, 'S', substr(comment, 1, position(' ' in comment)) as chunk, 'E'";
     $query .= "     FROM rawExp";
     $query .= "     LEFT JOIN chipRun using (exp_id)";
@@ -91 +92 @@
     $query .= "     AND (camProcessedExp.quality IS NULL or camProcessedExp.quality = 0)";
 #   $query .= "     GROUP BY comment, filter, substr(comment, 1, position(' ' in comment))";
-#    $query .= "   ) AS tableChunks GROUP BY object, filter, chunk";
+    $query .= "   ) AS tableChunks GROUP BY object, filter, chunk";
 #    $query .= " ) AS tableVisits GROUP BY Nvisit";
 
@@ -392 +393 @@
     $diffExpect = 0;
     while (@row = $result->fetchrow_array()) {
-        # printf "%8s %15s : ", $row[0], $row[1]; # c, count(*)
+        printf "%8s %15s : ", $row[0], $row[1]; # c, count(*)
         if ($row[0] == 1) {
             if ($row[1] == 1) { $verb = "tuple has"; } else { $verb = "tuples have"; }
@@ -464 +465 @@
     # my $result = &mysql_select ("");
     my $query = "";
-    $query .= "     SELECT exp_name, object, filter, substr(comment, 1, position(' ' in comment)) as chunk";
+    $query .= "     SELECT 'EXAMPLE', exp_name, object, filter, substr(comment, 1, position(' ' in comment)) as chunk";
     $query .= "     FROM rawExp";
     $query .= "     LEFT JOIN chipRun using (exp_id)";
@@ -491 +492 @@
     $query .= " WHERE data_group like '%$dataGroupDate'";
     $query .= " AND label NOT LIKE '%WS.nigh%'";
+    $query .= " AND label NOT LIKE 'goto_cleaned'";
     $query .= " GROUP BY label, state";
 
@@ -519 +521 @@
     $query .= "    WHERE data_group LIKE '%$dataGroupDate%'";
     $query .= "    AND label NOT LIKE '%WS.nigh%'";
+    $query .= "    AND label NOT LIKE 'goto_cleaned'";
     $query .= "    AND (quality IS NULL OR quality = 0)";
     $query .= "    GROUP BY label, state, diff_id";
@@ -542 +545 @@
     $query .= " WHERE data_group LIKE '%$dataGroupDate'";
     $query .= " AND diffRun.label NOT LIKE '%WS.nigh%'";
+    $query .= " AND diffRun.label NOT LIKE 'goto_cleaned'";
     $query .= " GROUP BY diffRun.label, publishRun.state";
 
@@ -762 +766 @@
 }
 
+# count the number of chunks, and their status:
+if ($verbose or $chunks) {
+    print "Current status of SS chunks:\n";
+
+    #This is a long query, but it efficiently gathers all the necessary information
+    my $query = "SELECT groupchunk.chunk,groupchunk.Nquad_firstvisit,group4.Nquad_fourvisits,groupdiff.Ndiffs FROM";
+    $query .= "   (";
+    $query .= "     SELECT dateobs,comment,substr(comment, 1, position(' ' in comment)) AS chunk,count(*) AS Nquad_firstvisit";
+    $query .= "     FROM rawExp";
+    $query .= "     WHERE dateobs LIKE '$date%' AND obs_mode LIKE '%SS%'";
+    $query .= "     AND obs_mode LIKE '%SS%' AND obs_mode NOT LIKE 'ENGINEERING'";
+    $query .= "     AND comment LIKE '%visit 1%'";
+    $query .= "     GROUP BY chunk";
+    $query .= "   ) AS groupchunk";
+    $query .= "   LEFT JOIN";
+    $query .= "   (";
+    $query .= "     SELECT groupobj.chunk,count(*) AS Nquad_fourvisits FROM ";
+    $query .= "       (SELECT substr(comment, 1, position(' ' in comment)) AS chunk,count(*) AS Nvisits";
+    $query .= "         FROM rawExp";
+    $query .= "         WHERE obs_mode LIKE '%' AND dateobs LIKE '$date%' AND obs_mode like '%SS%'";
+    $query .= "         AND obs_mode NOT LIKE 'ENGINEERING' AND comment LIKE '%visit%'";
+    $query .= "         GROUP BY object, filter, chunk";
+    $query .= "       ) AS groupobj";
+    $query .= "     WHERE groupobj.Nvisits = 4 ";
+    $query .= "     GROUP BY groupobj.chunk";
+    $query .= "   ) AS group4";
+    $query .= "   on groupchunk.chunk=group4.chunk";
+    $query .= "   LEFT JOIN";
+    $query .= "   (";
+    $query .= "     SELECT substr(expall.comment, 1, position(' ' in expall.comment)) AS chunk,count(*) AS Ndiffs FROM ";
+    $query .= "       (SELECT exp_name,comment,dateobs";
+    $query .= "         FROM rawExp ";
+    $query .= "         WHERE dateobs LIKE '$date%' AND obs_mode LIKE '%SS%' AND obs_mode NOT LIKE 'ENGINEERING'";
+    $query .= "       ) AS expall ";
+    $query .= "     LEFT JOIN";
+    $query .= "     (SELECT * FROM ";
+    $query .= "       (";
+    $query .= "         (SELECT exp_name FROM ";
+    $query .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp1=warp_id)";
+    $query .= "           JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
+    $query .= "           WHERE rawExp.dateobs LIKE '$date%' AND diffRun.reduction LIKE 'SWEETSPOT'";
+    $query .= "           AND stack2 IS NULL AND diffRun.diff_id=stage_id";
+    $query .= "         )  ";
+    $query .= "         UNION ";
+    $query .= "         (SELECT exp_name FROM";
+    $query .= "           diffRun JOIN diffInputSkyfile USING (diff_id) JOIN publishRun ON (diff_id = stage_id) JOIN warpRun ON (warp2=warp_id)";
+    $query .= "           JOIN fakeRun USING (fake_id) JOIN camRun USING (cam_id) JOIN camProcessedExp USING (cam_id) JOIN chipRun USING (chip_id) JOIN rawExp USING (exp_id)";
+    $query .= "           WHERE rawExp.dateobs LIKE '$date%' AND diffRun.reduction LIKE 'SWEETSPOT'";
+    $query .= "           AND stack2 IS NULL AND diffRun.diff_id=stage_id";
+    $query .= "         )";
+    $query .= "       ) AS diffgather ";
+    $query .= "       GROUP BY diffgather.exp_name";
+    $query .= "     ) AS exppub";
+    $query .= "     ON expall.exp_name=exppub.exp_name";
+    $query .= "     WHERE exppub.exp_name=expall.exp_name";
+    $query .= "     GROUP BY chunk ";
+    $query .= "   ) AS groupdiff";
+    $query .= "   ON groupchunk.chunk=groupdiff.chunk";
+
+    my $result = &mysql_select ($query);
+
+    printf "%23s | %23s | %23s | %23s |\n", "chunk", "Nquad_firstvisit done", "Nquad_fourvisits done", "Ndiffs done"; 
+    while (@row = $result->fetchrow_array()) {
+          if ($row[1] ne $row[2]) {
+            if ($row[3] ne ($row[1]*4)) {printf "%23s | %23s | %23s | %19s%4s | %23s\n", $row[0], $row[1], $row[2], $row[3], "/?  ", "chunk not finished";}  # chunk    Nquad_firstvisit        Nquad_fourvisits        Ndiffs
+            if ($row[3] eq ($row[1]*4)) {printf "%23s | %23s | %23s | %19s%4s | %23s\n", $row[0], $row[1], $row[2], $row[3], "/?  ", "chunk done";}          # chunk    Nquad_firstvisit        Nquad_fourvisits        Ndiffs
+          }
+          if ($row[1] eq $row[2]) {
+            if ($row[3] ne ($row[1]*4)) {printf "%23s | %23s | %23s | %19s%1s%3s | %23s\n", $row[0], $row[1], $row[2], $row[3],"/",$row[1]*4, "chunk not finished";}  # chunk   Nquad_firstvisit        Nquad_fourvisits        Ndiffs
+            if ($row[3] eq ($row[1]*4)) {printf "%23s | %23s | %23s | %19s%1s%3s | %23s\n", $row[0], $row[1], $row[2], $row[3],"/",$row[1]*4, "chunk done";}          # chunk   Nquad_firstvisit        Nquad_fourvisits        Ndiffs
+          }
+    }
+
+    print "\n";
+}
+
 exit 0;