IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 32265


Ignore:
Timestamp:
Aug 31, 2011, 5:18:37 PM (15 years ago)
Author:
watersc1
Message:

Updates to finish migration of summarization code into nebulous outputs

Location:
trunk/tools/diskspace
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/diskspace/merge_counts_and_sizes.pl

    r32230 r32265  
    22
    33use Getopt::Std;
     4use DateTime;
     5use PS::IPP::Config;
    46
    5 getopts('I',\%opt);
     7getopts('N',\%opt);
    68
    7 open(S,"summary.dat") || die "Cannot find summary.dat\n";
     9my $ipprc = PS::IPP::Config->new("GPC1");
     10my $nebcon = $ipprc->nebulous();
     11
     12my @lt = gmtime;
     13my $dt = DateTime->now;
     14$dt->subtract( days => $lt[6] );
     15my $datestring = $dt->ymd;
     16
     17my $summary_file = "neb:///ipp_diskspace/${datestring}/summary.dat";
     18my $S_real_file = $ipprc->file_resolve($summary_file);
     19open(S,$S_real_file) || die "Cannot find summary.dat\n";
    820
    921while (<S>) {
    … …  
    1931close(S);
    2032
    21 if (exists($opt{I})) {
    22     open(C,"run_im_counts.dat") || die "Cannot find run_counts.dat\n";
    23 }
    24 else {
    25     open(C,"run_counts.dat") || die "Cannot find run_counts.dat\n";
    26 }
     33my $count_file = "neb:///ipp_diskspace/${datestring}/run_im_counts.dat";
     34my $C_real_file = $ipprc->file_resolve($count_file);
     35open(C,$C_real_file) || die "Cannot find run_counts.dat\n";
    2736while(<C>) {
    2837    chomp;
    … …  
    3746close(C);
    3847
    39 if (exists($opt{I})) {
    40     open(M,"mappings_im.dat") || die "Cannot find mappings.dat\n";
    41 }
    42 else {
    43     open(M,"mappings.dat") || die "Cannot find mappings.dat\n";
    44 }
     48my $mappings_file = "neb:///ipp_diskspace/mappings_im.dat";
     49my $M_real_file = $ipprc->file_resolve($mappings_file);
     50open(M,$M_real_file) || die "Cannot find mappings.dat\n";
    4551while (<M>) {
    4652    chomp;
    … …  
    5763}
    5864close(M);
     65
    5966foreach $stage (keys (%size)) {
    6067    foreach $state (keys (%{ $count{$stage} })) {
    … …  
    6875
    6976foreach $stage (keys (%size)) {
    70 #    unless ($stage eq 'RAW') { next; }
    7177    foreach $state (keys (%{ $count{$stage} })) {
    7278        $VpN{$stage}{$state} = 0;
    7379        foreach $product (keys (%{ $size{$stage} })) {
    7480            $N{$stage}{$product} += $map{$stage}{$product}{$state} * $count{$stage}{$state};
    75 #           if ($N{$stage}{$product} > 0) {
    76 #               $VpN{$stage}{$state} += $size{$stage}{$product} / $N{$stage}{$product};
    77 #           }
    78 
    7981        }               
    8082    }
    … …  
    8991}
    9092
     93my $out_fh;
     94
     95if (exists($opt{N})) {
     96    my $dt = DateTime->now;
     97
     98    my @lt = gmtime;
     99    my $dt_data = DateTime->now;
     100    $dt->subtract( days => $lt[6]);
     101    my $datestring = $dt->ymd;
     102    my $ipprc = PS::IPP::Config->new("GPC1");
     103    my $nebcon = $ipprc->nebulous();
     104
     105    my $outfile = "neb:///ipp_diskspace/${datestring}/usage_report.dat";
     106
     107    if ($ipprc->file_exists($outfile)) {
     108        my $U_file = $ipprc->file_resolve($outfile);
     109        open($out_fh,">$U_file");
     110    }
     111    else {
     112        $out_fh = $ipprc->file_create_open($outfile);
     113    }
     114}
     115
    91116foreach $stage (sort (keys %VpN)) {
    92 #    unless ($stage eq 'RAW') { next; }
    93 
    94117    foreach $state (sort (keys %{ $VpN{$stage} })) {
    95118        unless(exists($count{$stage}{$state})) {
    … …  
    124147        printf("%-8s %-18s %-10s %14.4f %12.6f %9d\n",
    125148                $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state});
     149        if (exists($opt{N})) {
     150            printf($out_fh "%-8s %-18s %-10s %14.4f %12.6f %9d\n",
     151                   $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state});
     152        }
    126153    }
    127154}
    … …  
    130157    $sum += $labels{$l};
    131158    print STDERR "\t$l  \t$labels{$l}\n";
     159    if (exists($opt{N})) {
     160        print $out_fh "\t$l  \t$labels{$l}\n";
     161    }
    132162}
    133163print STDERR "\tTOTAL    \t$sum\n";
    134164print STDERR "\tEXP_TOT  \t$total\n";
     165
     166if (exists($opt{N})) {
     167    print $out_fh "\tTOTAL    \t$sum\n";
     168    print $out_fh "\tEXP_TOT  \t$total\n";
     169    close($out_fh);
     170}

  • trunk/tools/diskspace/nebulous_data.pl

    r32233 r32265  
    5252my $nebcon = $ipprc->nebulous();
    5353
    54 my @lt = localtime;
    55 $lt[5] += 1900;
    56 $lt[4] += 1;
    57 $lt[3] -= $lt[6];
    58 my $datestring = sprintf("%04d-%02d-%02d",$lt[5],$lt[4],$lt[3]);
     54my @lt = gmtime;
     55$dt = DateTime->now;
     56$dt->subtract( days => $lt[6]);
     57my $datestring = $dt->ymd;
    5958my $outfile = "neb:///ipp_diskspace/${datestring}/${neb}.neb_usage.dat";
    6059my $out_fh = $ipprc->file_create_open($outfile);

  • trunk/tools/diskspace/nebulous_wiki.pl

    r32230 r32265  
    22
    33use DateTime;
    4 $dt = DateTime->now;
     4use Getopt::Std;
     5use PS::IPP::Config;
    56
    67
    7 @files = sort (<*.size.dat>);
     8$opt{S} = 0;
     9getopts('S',\%opt);
     10
     11my $dt = DateTime->now;
     12
     13my @lt = gmtime;
     14my $dt_data = DateTime->now;
     15$dt->subtract( days => $lt[6]);
     16my $datestring = $dt->ymd;
     17
     18@files = sort ( split /\n/, `neb-ls -p neb:///ipp_diskspace/${datestring}/%neb_usage.dat`);
    819
    920foreach $f (@files) {
    1021    $volume = $f;
    11     $volume =~ s/.size.dat//;
     22    $volume =~ s/.*://;
     23    $volume =~ s/.neb_usage.dat//;
    1224    open(F,$f);
    1325    while(<F>) {
    … …  
    3749$presep = '||';
    3850
    39 print "${presep}Volume${sep}";
    40 foreach $w (sort (keys %sizes)) {
    41     print "$w$sep";
    42 }
    43 print "\n";
    44 foreach $v (sort (keys %sum)) {
    45     print "${presep}${v}${sep}";
     51if ($opt{S} == 0) {
     52    print "${presep}Volume${sep}";
    4653    foreach $w (sort (keys %sizes)) {
    47         $val = $vol{$v}{$w} / 1024 / 1024 / 1024;
     54        print "$w$sep";
     55    }
     56    print "\n";
     57    foreach $v (sort (keys %sum)) {
     58        print "${presep}${v}${sep}";
     59        foreach $w (sort (keys %sizes)) {
     60            $val = $vol{$v}{$w} / 1024 / 1024 / 1024;
     61            printf "%.2f%s",$val,$sep;
     62        }
     63        $val = $sum{$v} / 1024 / 1024 / 1024;
     64        printf "%.2f%s",$val,$sep;
     65        print "\n";
     66    }
     67    $date = $dt->ymd;
     68    print "${presep}${date}${sep}";
     69    foreach $w (sort (keys %sizes)) {
     70        $val = $sizes{$w} / 1024 / 1024 / 1024;
    4871        printf "%.2f%s",$val,$sep;
    4972    }
    50     $val = $sum{$v} / 1024 / 1024 / 1024;
    51     printf "%.2f%s",$val,$sep;
    5273    print "\n";
    5374}
    54 $date = $dt->ymd;
    55 print "${presep}${date}${sep}";
    56 foreach $w (sort (keys %sizes)) {
    57     $val = $sizes{$w} / 1024 / 1024 / 1024;
    58     printf "%.2f%s",$val,$sep;
     75else {
     76    my $ipprc = PS::IPP::Config->new("GPC1");
     77    my $nebcon = $ipprc->nebulous();
     78
     79    my $outfile = "neb:///ipp_diskspace/${datestring}/summary.dat";
     80    my $out_fh;
     81    if ($ipprc->file_exists($outfile)) {
     82        my $S_file = $ipprc->file_resolve($outfile);
     83        open($out_fh,">$S_file");
     84    }
     85    else {
     86        $out_fh = $ipprc->file_create_open($outfile);
     87    }
     88    foreach $w (sort (keys %sizes)) {
     89        $val = $sizes{$w} / 1024 / 1024 / 1024;
     90        print $out_fh "$w $val\n";
     91    }
     92    close($out_fh);
    5993}
    60 print "\n";
    6194
    6295

  • trunk/tools/diskspace/run_counter.pl

    r32230 r32265  
    1 #! /usr/local/bin/perl -w
     1#! /usr/bin/env perl
     2use warnings;
    23
     4use DateTime;
     5use Sys::Hostname;
     6use PS::IPP::Config;
    37use DBI;
    4 use Getopt::Std;
    58
    6 getopts('I',\%opt);
    79# Set up the database
    810use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
    … …  
    4143$im_query = 'select data_state,count(data_state) AS N from TABLETABLE GROUP BY data_state';
    4244
    43 if (exists($opt{I})) {
    44     foreach $k (keys %im_stages) {
    45         $stages{$k} = $im_stages{$k};
    46     }
     45foreach $k (keys %im_stages) {
     46    $stages{$k} = $im_stages{$k};
    4747}
     48
     49my $ipprc = PS::IPP::Config->new("GPC1");
     50my $nebcon = $ipprc->nebulous();
     51
     52my @lt = gmtime;
     53$dt = DateTime->now;
     54$dt->subtract( days => $lt[6]);
     55my $datestring = $dt->ymd;
     56my $outfile = "neb:///ipp_diskspace/${datestring}/run_im_counts.dat";
     57my $out_fh = $ipprc->file_create_open($outfile);
    4858
    4959foreach $s (keys %stages) {
    5060    my $query = $base_query;
    51     if (exists($opt{I}) && exists($im_stages{$s})) {
     61    if (exists($im_stages{$s})) {
    5262        $query = $im_query;
    5363    }
    … …  
    5767    foreach $rr (@{ $dr }) {
    5868        my ($state,$count) = @{ $rr };
    59         print "$s $state $count\n";
     69        print $out_fh "$s $state $count\n";
    6070    }
    6171}
    62            
     72close($out_fh);
     73
Note: See TracChangeset for help on using the changeset viewer.