Index: /tags/ipp-20110622/tools/diskspace/generate_mappings.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/generate_mappings.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/generate_mappings.pl (revision 32253) @@ -0,0 +1,38 @@ +#! /usr/local/bin/perl -w +my $summary = shift; +my $runs = shift; + +open(S,"$summary") || die "Cannot open summary $summary\n"; +while () { + chomp; + unless ($_ =~ /^#/) { + ($stage_product,$size_in_gb) = split /\s+/; + ($stage,$product) = split /_/, $stage_product; + + $size{$stage}{$product} = $size_in_gb; + } +} +close(S); +close(S); + +open(C,"$runs") || die "Cannot find runs $runs\n"; +while() { + chomp; + unless ($_ =~ /^#/) { + ($stage,$state,$N) = split /\s/; + if ($state eq '') { + $state = 'NULL'; + } + $count{$stage}{$state} = $N; + } +} +close(C); + +foreach $stage (keys (%size)) { + foreach $product (keys (%{ $size{$stage} })) { + foreach $state (keys (%{ $count{$stage} })) { + print "$stage $product $state\n"; + } + } +} + Index: /tags/ipp-20110622/tools/diskspace/merge_counts_and_sizes.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/merge_counts_and_sizes.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/merge_counts_and_sizes.pl (revision 32253) @@ -0,0 +1,134 @@ +#! /usr/local/bin/perl -w + +use Getopt::Std; + +getopts('I',\%opt); + +open(S,"summary.dat") || die "Cannot find summary.dat\n"; + +while () { + chomp; + unless ($_ =~ /^#/) { + ($stage_product,$size_in_gb) = split /\s+/; + ($stage,$product) = split /_/, $stage_product; + + $size{$stage}{$product} = $size_in_gb; + $total += $size_in_gb; + } +} +close(S); + +if (exists($opt{I})) { + open(C,"run_im_counts.dat") || die "Cannot find run_counts.dat\n"; +} +else { + open(C,"run_counts.dat") || die "Cannot find run_counts.dat\n"; +} +while() { + chomp; + unless ($_ =~ /^#/) { + ($stage,$state,$N) = split /\s/; + if ($state eq '') { + $state = 'NULL'; + } + $count{$stage}{$state} = $N; + } +} +close(C); + +if (exists($opt{I})) { + open(M,"mappings_im.dat") || die "Cannot find mappings.dat\n"; +} +else { + open(M,"mappings.dat") || die "Cannot find mappings.dat\n"; +} +while () { + chomp; + unless ($_ =~ /^#/) { + ($stage,$product,$state) = split /\s+/; + $map{$stage}{$product}{$state} = 1; + if ($stage eq 'RAW' && $product eq 'FITS') { + if (($state eq 'full')||($state eq 'pending_burntool')||($state eq 'keep')||($state eq 'compressed')||($state eq 'error_compressed')|| + ($state eq 'goto_lossy')||($state eq 'check_burntool')) { + $map{$stage}{$product}{$state} = 2; + } + } + } +} +close(M); +foreach $stage (keys (%size)) { + foreach $state (keys (%{ $count{$stage} })) { + foreach $product (keys (%{ $size{$stage} })) { + unless (exists($map{$stage}{$product}{$state})) { + $map{$stage}{$product}{$state} = 0; + } + } + } +} + +foreach $stage (keys (%size)) { +# unless ($stage eq 'RAW') { next; } + foreach $state (keys (%{ $count{$stage} })) { + $VpN{$stage}{$state} = 0; + foreach $product (keys (%{ $size{$stage} })) { + $N{$stage}{$product} += $map{$stage}{$product}{$state} * $count{$stage}{$state}; +# if ($N{$stage}{$product} > 0) { +# $VpN{$stage}{$state} += $size{$stage}{$product} / $N{$stage}{$product}; +# } + + } + } + foreach $product (keys (%{ $size{$stage} })) { + foreach $state (keys (%{ $count{$stage} })) { + if ($N{$stage}{$product} > 0) { + $VpN{$stage}{$state} += $map{$stage}{$product}{$state} * $size{$stage}{$product} / $N{$stage}{$product}; + } +# print STDERR ">>$stage $product $state $map{$stage}{$product}{$state} $N{$stage}{$product} $count{$stage}{$state} $size{$stage}{$product} $VpN{$stage}{$state}\n"; + } + } +} + +foreach $stage (sort (keys %VpN)) { +# unless ($stage eq 'RAW') { next; } + + foreach $state (sort (keys %{ $VpN{$stage} })) { + unless(exists($count{$stage}{$state})) { + $count{$stage}{$state} = 0; + } + $V = $VpN{$stage}{$state} * $count{$stage}{$state}; + + if ($stage eq 'RAW') { + $label = "PERMANENT"; + } + else { + if (($state eq 'goto_cleaned')|| + ($state eq 'goto_purged')|| + ($state eq 'goto_scrubbed')|| + ($state eq 'update')) { + $label = "TRANSIENT"; + } + elsif (($state eq 'full')&&( + ($stage eq 'CHIP')|| + ($stage eq 'WARP')|| + ($stage eq 'DIFF')|| + ($stage eq 'DIST')|| + ($stage eq 'DESTREAK') + )) { + $label = 'TRANSIENT'; + } + else { + $label = 'PRODUCT'; + } + } + $labels{$label} += $V; + printf("%-8s %-18s %-10s %14.4f %12.6f %9d\n", + $stage,$state,$label,$V,$VpN{$stage}{$state},$count{$stage}{$state}); + } +} +$sum = 0; +foreach $l (keys %labels) { + $sum += $labels{$l}; + print STDERR "\t$l \t$labels{$l}\n"; +} +print STDERR "\tTOTAL \t$sum\n"; +print STDERR "\tEXP_TOT \t$total\n"; Index: /tags/ipp-20110622/tools/diskspace/nebulous_data.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/nebulous_data.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/nebulous_data.pl (revision 32253) @@ -0,0 +1,105 @@ +#! /usr/bin/env perl +use warnings; + +use Sys::Hostname; +use PS::IPP::Config; +%stages = ("RAW" => '.ota', + "DIST" => 'dist', + "DESTREAK" => 'destreak', + "CHIP" => '.ch.', + "CAMERA" => '.cm.', + "WARP" => '.wrp.', + "DIFF" => '.dif.', + "STACK" => '.stk.', + "MAGIC" => '.mgc.', + "DETREND" => '.det' + ); + +@products = ('.log','.trace','.mdc$','.stats$', + '.smf$','.cmf$', + '.burn.tbl$','.tgz$', + '.b1.fits$','.b2.fits', + '.b1.jpg$','.b2.jpg$', + '.b1.jpeg$','.b2.jpeg$', + '.png$', + '.mdl.fits$','.ptn$','.psf$', + '.skycell$','kernel$', + '.mk.fits$','.mask.fits$','.wt.fits$', + '.exp.fits$','.num.fits$','.expwt.fits$', + '.fits.fz$', + '.fits$'); + +%product_names = ('.log' => "LOG",'.trace' => "TRACE",'.mdc$' => "MDC",'.stats$' => "STATS", + '.smf$' => "CATALOG",'.cmf$' => "CATALOG", + '.burn.tbl$' => "BTTABLE",'.tgz$' => "BUNDLE", + '.b1.fits$' => "B1FITS",'.b2.fits' => "B2FITS", + '.b1.jpg$' => "B1JPG",'.b2.jpg$' => "B2JPG", + '.b1.jpeg$' => "B1JPG",'.b2.jpeg$' => "B2JPG", + '.png$' => "PNG", + '.mdl.fits$' => "MDL",'.ptn$' => "PTN" ,'.psf$' => "PSF", + '.skycell$' => "SKYCELL",'kernel$' => "KERNEL", + '.mk.fits$' => "MASK",'.mask.fits$' => "MASK",'.wt.fits$' => "WEIGHT", + '.exp.fits$' => "MISCFITS",'.num.fits$' => "MISCFITS",'.expwt.fits$' => "MISCFITS", + '.fits.fz$' => "FITSCOMP", + '.fits$' => "FITS"); + +my $hostname = hostname(); +my $neb = shift(@ARGV); +unless(defined($neb)) { + $neb = "${hostname}.0"; +} +my $ipprc = PS::IPP::Config->new("GPC1"); +my $nebcon = $ipprc->nebulous(); + +my @lt = localtime; +$lt[5] += 1900; +$lt[4] += 1; +$lt[3] -= $lt[6]; +my $datestring = sprintf("%04d-%02d-%02d",$lt[5],$lt[4],$lt[3]); +my $outfile = "neb:///ipp_diskspace/${datestring}/${neb}.neb_usage.dat"; +my $out_fh = $ipprc->file_create_open($outfile); + +my $neb_dir = "/data/${neb}/nebulous/"; + +open(FIND,"find $neb_dir -type f -printf \"%f %s\n\" |") || die "Cannot run find.\n"; +while () { + chomp; + ($file,$size) = split /\s+/; + + my $stage = 'OTHER'; + my $product = 'UNKNOWN'; + + foreach $k (keys %stages) { + if ($file =~ /$stages{$k}/) { + $stage = $k; + last; + } + } + foreach $k (@products) { + if ($file =~ /$k/) { + $product = $product_names{$k}; + unless (defined($product)) { + print "$k $product\n"; + die; + } + last; + } + } + + $count{$stage}{$product} += 1; + $volume{$stage}{$product} += $size; +} +close(FIND); + + + +foreach $s (sort (keys %count)) { + foreach $p (sort (keys %{ $count{$s} })) { + print $out_fh "$s $p $count{$s}{$p} $volume{$s}{$p}\n"; + } +} + +close($out_fh); + + + Index: /tags/ipp-20110622/tools/diskspace/nebulous_wiki.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/nebulous_wiki.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/nebulous_wiki.pl (revision 32253) @@ -0,0 +1,63 @@ +#! /usr/local/bin/perl -w + +use DateTime; +$dt = DateTime->now; + + +@files = sort (<*.size.dat>); + +foreach $f (@files) { + $volume = $f; + $volume =~ s/.size.dat//; + open(F,$f); + while() { + chomp; + ($stage,$product,$count,$size) = split /\s+/; + $what = "${stage}_${product}"; + $num{$volume}{$what} = $count; + $vol{$volume}{$what} = $size; + + $sum{$volume} += $size; + $sizes{$what} += $size; + } + close(F); +} +foreach $v (keys %sum) { + foreach $w (keys %sizes) { + unless (exists($vol{$v}{$w})) { + $num{$v}{$w} = 0; + } + unless (exists($vol{$v}{$w})) { + $vol{$v}{$w} = 0; + } + } +} + +$sep = '||'; +$presep = '||'; + +print "${presep}Volume${sep}"; +foreach $w (sort (keys %sizes)) { + print "$w$sep"; +} +print "\n"; +foreach $v (sort (keys %sum)) { + print "${presep}${v}${sep}"; + foreach $w (sort (keys %sizes)) { + $val = $vol{$v}{$w} / 1024 / 1024 / 1024; + printf "%.2f%s",$val,$sep; + } + $val = $sum{$v} / 1024 / 1024 / 1024; + printf "%.2f%s",$val,$sep; + print "\n"; +} +$date = $dt->ymd; +print "${presep}${date}${sep}"; +foreach $w (sort (keys %sizes)) { + $val = $sizes{$w} / 1024 / 1024 / 1024; + printf "%.2f%s",$val,$sep; +} +print "\n"; + + + Index: /tags/ipp-20110622/tools/diskspace/run_counter.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/run_counter.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/run_counter.pl (revision 32253) @@ -0,0 +1,62 @@ +#! /usr/local/bin/perl -w + +use DBI; +use Getopt::Std; + +getopts('I',\%opt); +# Set up the database +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ippdb01'; +$dbuser = 'ippuser'; +$dbpass = 'ippuser'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + +%stages = ("RAW" => 'rawExp', + "CHIP" => 'chipRun', + 'CAMERA' => 'camRun', + 'WARP' => 'warpRun', + 'DIFF' => 'diffRun', + 'STACK' => 'stackRun', + 'DIST' => 'distRun', + 'DESTREAK' => 'magicDSRun', + 'MAGIC' => 'magicRun', + 'DETREND' => 'detRun'); + +$base_query = 'select state,count(state) AS N from TABLETABLE GROUP BY state'; + + +%im_stages = ("RAW" => 'rawImfile', + "CHIP" => 'chipProcessedImfile', + "WARP" => 'warpSkyfile', + "DIFF" => 'diffSkyfile' + ); + + +$im_query = 'select data_state,count(data_state) AS N from TABLETABLE GROUP BY data_state'; + +if (exists($opt{I})) { + foreach $k (keys %im_stages) { + $stages{$k} = $im_stages{$k}; + } +} + +foreach $s (keys %stages) { + my $query = $base_query; + if (exists($opt{I}) && exists($im_stages{$s})) { + $query = $im_query; + } + $query =~ s/TABLETABLE/$stages{$s}/; + print STDERR "$s $query\n"; + my $dr = $db->selectall_arrayref( $query ); + foreach $rr (@{ $dr }) { + my ($state,$count) = @{ $rr }; + print "$s $state $count\n"; + } +} + Index: /tags/ipp-20110622/tools/diskspace/sizes_from_counts.pl =================================================================== --- /tags/ipp-20110622/tools/diskspace/sizes_from_counts.pl (revision 32253) +++ /tags/ipp-20110622/tools/diskspace/sizes_from_counts.pl (revision 32253) @@ -0,0 +1,111 @@ +#! /usr/local/bin/perl -w +my $count_list = shift; + +while() { + chomp; + ($stage,$state,$label,undef,$rate,$count) = split /\s+/; + $data{$stage}{$state}{L} = $label; + $data{$stage}{$state}{R} = $rate; + $cumulative{$label} = 0; +} +$total = 0; +open(D,"$count_list"); +while() { + chomp; + ($stage,$state,$count) = split /\s+/; + unless(exists($data{$stage}{$state})) { + next; + } + $label = $data{$stage}{$state}{L}; + $VpN = $data{$stage}{$state}{R}; + $V = $count * $VpN; + printf("%-8s %-18s %-10s %14.4f %12.6f %9d\n", + $stage,$state,$label,$V,$VpN,$count); + $cumulative{$label} += $V; + $total += $V; +} +foreach $l (keys %cumulative) { + print STDERR "\t$l \t$cumulative{$l}\n"; +} +print STDERR "\tSUM \t$total\n"; + + + + + +__DATA__ +CAMERA cleaned PRODUCT 296.4215 0.171540 1728 +CAMERA drop PRODUCT 0.1375 0.000671 205 +CAMERA full PRODUCT 96885.7234 0.388503 249382 +CAMERA goto_purged TRANSIENT 54.3905 0.388503 140 +CAMERA new PRODUCT 0.7720 0.000671 1151 +CAMERA purged PRODUCT 0.1715 0.171540 1 +CAMERA wait PRODUCT 0.7036 0.000671 1049 +CHIP cleaned PRODUCT 26777.0553 0.001840 14549732 +CHIP error_cleaned PRODUCT 2163.6842 0.105432 20522 +CHIP error_scrubbed PRODUCT 232.8603 0.039798 5851 +CHIP full TRANSIENT 39127.3368 0.119770 326686 +CHIP purged PRODUCT 514.9939 0.001840 279830 +CHIP scrubbed PRODUCT 11.0294 0.001840 5993 +CHIP update TRANSIENT 17.0801 0.105432 162 +DESTREAK censored PRODUCT 0.6473 0.029424 22 +DESTREAK cleaned PRODUCT 36.6759 0.000076 482521 +DESTREAK drop PRODUCT 0.0000 0.000000 1045 +DESTREAK error_cleaned PRODUCT 7.9444 0.029424 270 +DESTREAK failed_revert PRODUCT 0.0000 0.000000 1 +DESTREAK full TRANSIENT 4597.8025 0.029424 156261 +DESTREAK goto_cleaned TRANSIENT 9.7099 0.029424 330 +DESTREAK new PRODUCT 0.0000 0.000000 18 +DESTREAK new.hold PRODUCT 0.0000 0.000000 10 +DESTREAK restored PRODUCT 4.2959 0.029424 146 +DESTREAK todebug PRODUCT 0.0294 0.029424 1 +DESTREAK update TRANSIENT 351.6446 0.029424 11951 +DESTREAK wait PRODUCT 0.0000 0.000000 4 +DETREND drop PRODUCT 18.5816 0.120660 154 +DETREND register PRODUCT 89.4761 44.738033 2 +DETREND stop PRODUCT 15692.5025 49.976122 314 +DETREND wait PRODUCT 38.3698 0.120660 318 +DIFF cleaned PRODUCT 20727.1251 0.002301 9009076 +DIFF full TRANSIENT 48536.1541 0.147039 330091 +DIFF purged PRODUCT 362.3386 0.002301 157491 +DIFF update TRANSIENT 159.2223 0.125967 1264 +DIST cleaned PRODUCT 12.6523 0.000027 471189 +DIST drop PRODUCT 0.0000 0.000000 9070 +DIST full TRANSIENT 10451.6987 0.052522 198998 +DIST goto_cleaned TRANSIENT 27.8890 0.052522 531 +DIST new PRODUCT 0.0000 0.000000 2 +MAGIC cancel PRODUCT 0.0040 0.004025 1 +MAGIC censored PRODUCT 0.3139 0.004025 78 +MAGIC drop PRODUCT 0.0000 0.000000 438 +MAGIC full PRODUCT 771.2421 0.004025 191634 +MAGIC new PRODUCT 0.0000 0.000000 297 +MAGIC wait PRODUCT 0.0000 0.000000 9 +RAW NULL PERMANENT 0.0000 0.000394 0 +RAW check_burntool PERMANENT 19.4277 0.043269 449 +RAW compressed PERMANENT 720.8267 0.048541 14850 +RAW corrupt PERMANENT 0.0174 0.000471 37 +RAW error_compressed PERMANENT 292.1491 0.043346 6740 +RAW error_lossy PERMANENT 120.6904 0.027103 4453 +RAW full PERMANENT 607420.2114 0.043346 14013433 +RAW goto_lossy PERMANENT 130.8652 0.048541 2696 +RAW keep PERMANENT 24975.4561 0.043346 576194 +RAW lossy PERMANENT 24720.0428 0.005666 4363026 +RAW missing PERMANENT 0.0099 0.000394 25 +RAW pending_burntool PERMANENT 9220.6406 0.043269 213101 +RAW roc PERMANENT 73422.4326 0.021908 3351366 +STACK cleaned PRODUCT 86.8525 0.003704 23447 +STACK drop PRODUCT 0.0000 0.000000 8862 +STACK error_cleaned PRODUCT 914.9882 0.164241 5571 +STACK full PRODUCT 53913.0776 0.175869 306553 +STACK goto_cleaned TRANSIENT 14.7747 0.175889 84 +STACK goto_purged TRANSIENT 26.9356 0.164241 164 +STACK new PRODUCT 0.0000 0.000000 12917 +STACK purged PRODUCT 18.2914 0.003704 4938 +STACK wait PRODUCT 0.0000 0.000000 120 +WARP cleaned PRODUCT 6547.3188 0.000306 21418513 +WARP drop PRODUCT 0.0000 0.000000 1 +WARP error_cleaned PRODUCT 2210.2057 0.066446 33263 +WARP error_scrubbed PRODUCT 4.4519 0.066446 67 +WARP full TRANSIENT 36791.1577 0.066446 553697 +WARP purged PRODUCT 18.1011 0.000306 59215 +WARP scrubbed PRODUCT 11.6047 0.000306 37963