Changeset 32266

Timestamp:

Aug 31, 2011, 5:19:14 PM (15 years ago)

Author:

watersc1

Message:

Updates to finish migration of summarization code into nebulous outputs

Location:

tags/ipp-20110622/tools/diskspace

Files:

• . (modified) (1 prop)
• merge_counts_and_sizes.pl (modified) (8 diffs)
• nebulous_data.pl (modified) (1 diff)
• nebulous_wiki.pl (modified) (2 diffs)
• run_counter.pl (modified) (3 diffs)

tags/ipp-20110622/tools/diskspace/merge_counts_and_sizes.pl

@@ -2 +2 @@
 
 use Getopt::Std;
+use DateTime;
+use PS::IPP::Config;
 
-getopts('I',\%opt);
+getopts('N',\%opt);
 
-open(S,"summary.dat") || die "Cannot find summary.dat\n";
+my $ipprc = PS::IPP::Config->new("GPC1");
+my $nebcon = $ipprc->nebulous();
+
+my @lt = gmtime;
+my $dt = DateTime->now;
+$dt->subtract( days => $lt[6] );
+my $datestring = $dt->ymd;
+
+my $summary_file = "neb:///ipp_diskspace/${datestring}/summary.dat";
+my $S_real_file = $ipprc->file_resolve($summary_file);
+open(S,$S_real_file) || die "Cannot find summary.dat\n";
 
 while (<S>) {
@@ -19 +31 @@
 close(S);
 
-if (exists($opt{I})) {
-    open(C,"run_im_counts.dat") || die "Cannot find run_counts.dat\n";
-}
-else {
-    open(C,"run_counts.dat") || die "Cannot find run_counts.dat\n";
-}
+my $count_file = "neb:///ipp_diskspace/${datestring}/run_im_counts.dat";
+my $C_real_file = $ipprc->file_resolve($count_file);
+open(C,$C_real_file) || die "Cannot find run_counts.dat\n";
 while(<C>) {
     chomp;
@@ -37 +46 @@
 close(C);
 
-if (exists($opt{I})) {
-    open(M,"mappings_im.dat") || die "Cannot find mappings.dat\n";
-}
-else {
-    open(M,"mappings.dat") || die "Cannot find mappings.dat\n";
-}
+my $mappings_file = "neb:///ipp_diskspace/mappings_im.dat";
+my $M_real_file = $ipprc->file_resolve($mappings_file);
+open(M,$M_real_file) || die "Cannot find mappings.dat\n";
 while (<M>) {
     chomp;
@@ -57 +63 @@
 }
 close(M);
+
 foreach $stage (keys (%size)) {
     foreach $state (keys (%{ $count{$stage} })) {
@@ -68 +75 @@
 
 foreach $stage (keys (%size)) {
-#    unless ($stage eq 'RAW') { next; }
     foreach $state (keys (%{ $count{$stage} })) {
         $VpN{$stage}{$state} = 0;
         foreach $product (keys (%{ $size{$stage} })) {
             $N{$stage}{$product} += $map{$stage}{$product}{$state} * $count{$stage}{$state};
-#           if ($N{$stage}{$product} > 0) {
-#               $VpN{$stage}{$state} += $size{$stage}{$product} / $N{$stage}{$product};
-#           }
-
         }               
     }
@@ -89 +91 @@
 }
 
+my $out_fh;
+
+if (exists($opt{N})) {
+    my $dt = DateTime->now;
+
+    my @lt = gmtime;
+    my $dt_data = DateTime->now;
+    $dt->subtract( days => $lt[6]);
+    my $datestring = $dt->ymd;
+    my $ipprc = PS::IPP::Config->new("GPC1");
+    my $nebcon = $ipprc->nebulous();
+
+    my $outfile = "neb:///ipp_diskspace/${datestring}/usage_report.dat";
+
+    if ($ipprc->file_exists($outfile)) {
+        my $U_file = $ipprc->file_resolve($outfile);
+        open($out_fh,">$U_file");
+    }
+    else {
+        $out_fh = $ipprc->file_create_open($outfile);
+    }
+}
+
 foreach $stage (sort (keys %VpN)) {
-#    unless ($stage eq 'RAW') { next; }
-
     foreach $state (sort (keys %{ $VpN{$stage} })) {
         unless(exists($count{$stage}{$state})) {
@@ -124 +147 @@
         printf("%-8s %-18s %-10s %14.4f %12.6f %9d\n",
                 $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state});
+        if (exists($opt{N})) {
+            printf($out_fh "%-8s %-18s %-10s %14.4f %12.6f %9d\n",
+                   $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state});
+        }
     }
 }
@@ -130 +157 @@
     $sum += $labels{$l};
     print STDERR "\t$l  \t$labels{$l}\n";
+    if (exists($opt{N})) {
+        print $out_fh "\t$l  \t$labels{$l}\n";
+    }
 }
 print STDERR "\tTOTAL    \t$sum\n";
 print STDERR "\tEXP_TOT  \t$total\n";
+
+if (exists($opt{N})) {
+    print $out_fh "\tTOTAL    \t$sum\n";
+    print $out_fh "\tEXP_TOT  \t$total\n";
+    close($out_fh);
+}

tags/ipp-20110622/tools/diskspace/nebulous_data.pl

@@ -52 +52 @@
 my $nebcon = $ipprc->nebulous();
 
-my @lt = localtime;
-$lt[5] += 1900;
-$lt[4] += 1;
-$lt[3] -= $lt[6];
-my $datestring = sprintf("%04d-%02d-%02d",$lt[5],$lt[4],$lt[3]);
+my @lt = gmtime;
+$dt = DateTime->now;
+$dt->subtract( days => $lt[6]);
+my $datestring = $dt->ymd;
 my $outfile = "neb:///ipp_diskspace/${datestring}/${neb}.neb_usage.dat";
 my $out_fh = $ipprc->file_create_open($outfile);

tags/ipp-20110622/tools/diskspace/nebulous_wiki.pl

@@ -2 +2 @@
 
 use DateTime;
-$dt = DateTime->now;
+use Getopt::Std;
+use PS::IPP::Config;
 
 
-@files = sort (<*.size.dat>);
+$opt{S} = 0;
+getopts('S',\%opt);
+
+my $dt = DateTime->now;
+
+my @lt = gmtime;
+my $dt_data = DateTime->now;
+$dt->subtract( days => $lt[6]);
+my $datestring = $dt->ymd;
+
+@files = sort ( split /\n/, `neb-ls -p neb:///ipp_diskspace/${datestring}/%neb_usage.dat`);
 
 foreach $f (@files) {
     $volume = $f;
-    $volume =~ s/.size.dat//;
+    $volume =~ s/.*://;
+    $volume =~ s/.neb_usage.dat//;
     open(F,$f);
     while(<F>) {
@@ -37 +49 @@
 $presep = '||';
 
-print "${presep}Volume${sep}";
-foreach $w (sort (keys %sizes)) {
-    print "$w$sep";
-}
-print "\n";
-foreach $v (sort (keys %sum)) {
-    print "${presep}${v}${sep}";
+if ($opt{S} == 0) {
+    print "${presep}Volume${sep}";
     foreach $w (sort (keys %sizes)) {
-        $val = $vol{$v}{$w} / 1024 / 1024 / 1024;
+        print "$w$sep";
+    }
+    print "\n";
+    foreach $v (sort (keys %sum)) {
+        print "${presep}${v}${sep}";
+        foreach $w (sort (keys %sizes)) {
+            $val = $vol{$v}{$w} / 1024 / 1024 / 1024;
+            printf "%.2f%s",$val,$sep;
+        }
+        $val = $sum{$v} / 1024 / 1024 / 1024;
+        printf "%.2f%s",$val,$sep;
+        print "\n";
+    }
+    $date = $dt->ymd;
+    print "${presep}${date}${sep}";
+    foreach $w (sort (keys %sizes)) {
+        $val = $sizes{$w} / 1024 / 1024 / 1024;
         printf "%.2f%s",$val,$sep;
     }
-    $val = $sum{$v} / 1024 / 1024 / 1024;
-    printf "%.2f%s",$val,$sep;
     print "\n";
 }
-$date = $dt->ymd;
-print "${presep}${date}${sep}";
-foreach $w (sort (keys %sizes)) {
-    $val = $sizes{$w} / 1024 / 1024 / 1024;
-    printf "%.2f%s",$val,$sep;
+else {
+    my $ipprc = PS::IPP::Config->new("GPC1");
+    my $nebcon = $ipprc->nebulous();
+
+    my $outfile = "neb:///ipp_diskspace/${datestring}/summary.dat";
+    my $out_fh;
+    if ($ipprc->file_exists($outfile)) {
+        my $S_file = $ipprc->file_resolve($outfile);
+        open($out_fh,">$S_file");
+    }
+    else {
+        $out_fh = $ipprc->file_create_open($outfile);
+    }
+    foreach $w (sort (keys %sizes)) {
+        $val = $sizes{$w} / 1024 / 1024 / 1024;
+        print $out_fh "$w $val\n";
+    }
+    close($out_fh);
 }
-print "\n";
 
 

tags/ipp-20110622/tools/diskspace/run_counter.pl

@@ -1 @@
-#! /usr/local/bin/perl -w
+#! /usr/bin/env perl
+use warnings;
 
+use DateTime;
+use Sys::Hostname;
+use PS::IPP::Config;
 use DBI;
-use Getopt::Std;
 
-getopts('I',\%opt);
 # Set up the database
 use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
@@ -41 +43 @@
 $im_query = 'select data_state,count(data_state) AS N from TABLETABLE GROUP BY data_state';
 
-if (exists($opt{I})) {
-    foreach $k (keys %im_stages) {
-        $stages{$k} = $im_stages{$k};
-    }
+foreach $k (keys %im_stages) {
+    $stages{$k} = $im_stages{$k};
 }
+
+my $ipprc = PS::IPP::Config->new("GPC1");
+my $nebcon = $ipprc->nebulous();
+
+my @lt = gmtime;
+$dt = DateTime->now;
+$dt->subtract( days => $lt[6]);
+my $datestring = $dt->ymd;
+my $outfile = "neb:///ipp_diskspace/${datestring}/run_im_counts.dat";
+my $out_fh = $ipprc->file_create_open($outfile);
 
 foreach $s (keys %stages) {
     my $query = $base_query;
-    if (exists($opt{I}) && exists($im_stages{$s})) {
+    if (exists($im_stages{$s})) {
         $query = $im_query;
     }
@@ -57 +67 @@
     foreach $rr (@{ $dr }) {
         my ($state,$count) = @{ $rr };
-        print "$s $state $count\n";
+        print $out_fh "$s $state $count\n";
     }
 }
-           
+close($out_fh);
+