Changeset 29124 for branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm

Timestamp:

Sep 7, 2010, 9:16:10 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20100823

Files:

• . (modified) (1 prop)
• tools/czartool/CzarDb.pm (modified) (10 diffs)

branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm

@@ -1 @@
-#!/usr/bin/perl i-w
+#!/usr/bin/perl -w
 
 package czartool::CzarDb;
@@ -6 +6 @@
 use strict;
 
-my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
+my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
 
 use base 'czartool::MySQLDb';
@@ -119 +119 @@
 
     $query->execute;
+}
+
+###########################################################################
+#
+# Updates nightlyscience table
+#
+###########################################################################
+sub updateNightlyScience {
+    my ($self, $status) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM nightlyscience;
+SQL
+
+    $query->execute;
+
+    $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO nightlyscience
+        (status)
+        VALUES
+        ('$status');
+SQL
+
+       $query->execute;
 }
 
@@ -218 +242 @@
 ###########################################################################
 sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
-
-    my $dataFile = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_;
+
+    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -235 +260 @@
 SQL
 
-    if (!$query->execute) {return undef;}
+    if (!$query->execute) {return 0;}
 
     my $minProcessed = undef;
-
-    ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
-
-    if (!defined $minProcessed) {return undef;}
+    my ($_maxY, $_minY, $_timeDiff);
+    ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
+
+    if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;}
+    if ($_minY < ${$minY}) {${$minY} = $_minY;}
+    if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
+
+    if (!defined $minProcessed) {return 0;}
 
     $query = $self->{_db}->prepare(<<SQL);
@@ -256 +285 @@
     # loop round results
     while (my @row = $query->fetchrow_array()) {
-
         my ($timestamp, $pending, $faults, $processed) = @row;
-    #    print  "'$timestamp' '$pending' '$faults' '$processed'\n";
+
+        if ($isLog) {
+
+            $processed = log($processed + 1);
+            $pending = log($pending + 1);
+            $faults = log($faults + 1);
+        }
+
         print GNUDAT "$timestamp $pending $faults $processed\n";
     }
-    close(GNUDAT);
-
-    return $dataFile;
+
+    close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
+
+    return 1;
 }
 
@@ -491 +527 @@
 }
 
+
+###########################################################################
+#
+# Deletes all but one row per interval from all stage tables for all labels between the two dates
+#
+###########################################################################
+sub cleanupDateRange {
+    my ($self, $startDay, $endDay, $interval) = @_;
+
+    my $thisDay = $startDay;
+    my $quit = 0;
+    while(!$quit) {
+
+        if (!$self->isBefore($thisDay, $endDay)) {
+        
+            $quit = 1;
+        }
+
+        print "* Running cleanup for $thisDay with an interval of $interval\n";
+        $self->cleanupADay($thisDay, $interval);
+        $thisDay = $self->addInterval($thisDay, "1 DAY");
+
+    }
+}
+###########################################################################
+#
+# Deletes all but one row per interval from all stage tables for all labels between the provided day
+#
+###########################################################################
+sub cleanupADay {
+    my ($self, $startDay, $interval) = @_;
+
+    my $endDay =  $self->addInterval($startDay, "1 DAY");
+
+    my $labels = undef;
+    my $fromTime = $startDay;
+    my $toTime = undef;
+    my $totalDeleted = undef;
+    my $quit = 0;
+    while(!$quit) {
+
+        $toTime = $self->addInterval($fromTime, $interval);
+        if (!$self->isBefore($toTime, $endDay)) {
+        
+            $toTime = $endDay;
+            $quit = 1;
+        }
+   
+        my $stage = undef;
+        $totalDeleted = 0;
+        foreach $stage (@stages) {
+
+            if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;}
+
+            my $label = undef;
+            my $row = undef;
+            foreach $row ( @{$labels} ) {
+                my ($label) = @{$row};
+
+                my $query = $self->{_db}->prepare(<<SQL);
+                SELECT COUNT(*) 
+                    FROM $stage 
+                    WHERE timestamp > '$fromTime'
+                    AND timestamp <= '$toTime'
+                    AND label = '$label' 
+SQL
+
+                    $query->execute;
+
+                my $toDelete = scalar $query->fetchrow_array() - 1;
+                if ($toDelete < 1) {next;}
+
+                $query = $self->{_db}->prepare(<<SQL);
+                DELETE FROM $stage 
+                    WHERE timestamp > '$fromTime' 
+                    AND timestamp <= '$toTime' 
+                    AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete
+SQL
+                    $query->execute;
+
+                $totalDeleted += $toDelete;
+            }
+        }
+        print "   * Deleted $totalDeleted between $fromTime and  $toTime\n";
+        $fromTime = $toTime;
+    }
+}
+
+###########################################################################
+#
+# Returns an array of labels present during the provided time frame 
+#
+###########################################################################
+sub getLabelsInThisTimePeriod {
+    my ($self, $stage, $fromTime, $toTime, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT label 
+        FROM $stage 
+        WHERE timestamp > '$fromTime' 
+        AND timestamp <= '$toTime';
+SQL
+
+        if (!$query->execute) {
+
+            return 0;
+        }
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
 ###########################################################################
 #
@@ -500 +649 @@
 
     my $currentRevision = -1;
-    my $latestRevision = 10;
-
-    while ($currentRevision != $latestRevision) {
+
+    while (1) {
 
         $currentRevision = $self->getRevision();
@@ -517 +665 @@
         elsif ($currentRevision == 8) {$self->createRevision_9();}
         elsif ($currentRevision == 9) {$self->createRevision_10();}
+        elsif ($currentRevision == 10) {$self->createRevision_11();}
+        elsif ($currentRevision == 11) {$self->createRevision_12();}
+        else {last;}
     }
 }
@@ -841 +992 @@
 }
 
+#######################################################################################
+# 
+# Create revision 11 of the database 
+#
+#######################################################################################
+sub createRevision_11 {
+    my ($self) = @_;
+
+    print "* Creating revision 11 of '$self->{_dbName}'\n";
+
+    # same shape as other stage tables to enable easy update
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE burntool (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            label VARCHAR(128) DEFAULT "NONE", 
+            pending BIGINT NOT NULL,
+            processed BIGINT NOT NULL,
+            faults BIGINT NOT NULL);
+SQL
+
+      $query->execute;
+        $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX burntoolIndex ON burntool (timestamp, label);
+SQL
+
+      $query->execute;
+
+    $self->setRevision(11);
+}
+
+#######################################################################################
+# 
+# Create revision 12 of the database 
+#
+#######################################################################################
+sub createRevision_12 {
+    my ($self) = @_;
+
+    print "* Creating revision 12 of '$self->{_dbName}'\n";
+
+    # same shape as other stage tables to enable easy update
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE nightlyscience (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            status VARCHAR(128) DEFAULT "NONE"); 
+SQL
+
+    $query->execute;
+
+    $self->setRevision(12);
+}
+
 1;