IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Nov 26, 2010, 10:33:29 AM (16 years ago)
Author:
eugene
Message:

merge updates from trunk

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm

    r29618 r29834  
    44use warnings;
    55use strict;
     6
     7use File::Temp;
    68
    79my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
    … …  
    1618    my $class = shift;
    1719    my $self = {
     20        _gpc1Db => shift,
    1821        _czarDb => shift,
    1922        _dateFormat => shift,
    … …  
    5356}
    5457
     58
     59###########################################################################
     60#
     61# Figures out a suitable interval for rate plots
     62# Could use a function here, but instead we want to choose meaningful intervals
     63#
     64###########################################################################
     65sub getSuitableInterval {
     66    my ($self, $begin, $end, $interval) = @_;
     67
     68    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);
     69
     70
     71    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
     72    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
     73    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
     74    elsif ($self->{_czarDb}->isBefore($timeDiff, "24:00:00")) {${$interval} = "1 HOUR";} # under 1 day
     75    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
     76    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
     77    elsif ($self->{_czarDb}->isBefore($timeDiff, "72:00:00")) {${$interval} = "4 HOUR";} # under 3 days
     78    elsif ($self->{_czarDb}->isBefore($timeDiff, "96:00:00")) {${$interval} = "5 HOUR";} # under 4 days
     79    elsif ($self->{_czarDb}->isBefore($timeDiff, "144:00:00")) {${$interval} = "6 HOUR";} # under 6 days
     80    elsif ($self->{_czarDb}->isBefore($timeDiff, "168:00:00")) {${$interval} = "12 HOUR";} # under week
     81    elsif ($self->{_czarDb}->isBefore($timeDiff, "672:00:00")) {${$interval} = "1 DAY";} # under 2 week
     82    elsif ($self->{_czarDb}->isBefore($timeDiff, "2016:00:00")) {${$interval} = "1 WEEK";} # under 3 month
     83    else {${$interval} = "1 MONTH";}
     84
     85    return 1;
     86}
     87
    5588###########################################################################
    5689#
    … …  
    6194    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
    6295
    63     my $minX = 999999999;     
    64     my $maxX = -9999999999;
    65     my $timeDiff = -1;
    66 
     96    # stages
    6797    my $stages = undef;                 
    68 
    69     if (!$selectedStage) {$stages = \@allStages;}
    70     else {$stages = ["$selectedStage"]};       
    71 
    72     $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
    73 
    74     my $divX;
    75     my $timeFormat;
    76     $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
    77 
     98    if (!$selectedStage) {
     99        $stages = \@allStages;
     100        $selectedStage = 'all_stages';
     101    }
     102    else {$stages = ["$selectedStage"]};
     103
     104    # interval
     105    if (!$interval && !$self->getSuitableInterval($beginTime, $endTime, \$interval)) {return 0;}
     106
     107    # set up files
    78108    my %gnuplotFiles;
    79109    my $stage = undef;
    … …  
    89119        "set term $self->{_outputFormat};" .
    90120        "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
    91         #"set xdata time;" .
    92         "set timefmt \"$self->{_dateFormat}\";" .
    93        "set format x \"$timeFormat\";" .
    94         "set xtics \"$minX\", $divX, \"$maxX\";" .
    95 #        "set xrange [\"$minX\":\"$maxX\"];" .
    96121        "set boxwidth;" .
     122        "set xtic rotate by -90 scale 0;" .
    97123        "set style data histogram;" .
    98124        "set style histogram rowstacked;" .
    … …  
    124150    print GP ";\n";
    125151    close GP;
    126 }                                                   
     152
     153    # now delete temp dat files
     154    foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}
     155
     156    return 1;
     157}
     158
    127159###########################################################################
    128160#
    … …  
    168200    if ($numOfPlots == 0 ) {
    169201
    170         print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
     202#        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
    171203        return;
    172204    }
    … …  
    202234    my $outputFile = createImageFileName($self, $label, undef, "h", 0);
    203235
    204     my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table
    205         open (GNUDAT, ">$inputFile");
     236    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_h.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     237    open (GNUDAT, ">".$tmpFile->filename);
    206238
    207239    my ($processed, $pending, $faults);
    208240    my $stage = undef;
    209241    my $pendingMinusFaults = undef;
    210     my $maxY = 0;
    211     my $sum;
    212242    foreach $stage (@allStages) {
    213243
    214         $self->{_czarDb}->countProcessedPendingAndFaults(
     244        if (!$self->{_czarDb}->countProcessedPendingAndFaults(
    215245                $label,
    216246                $stage,
    … …  
    219249                \$processed,
    220250                \$pending,
    221                 \$faults);
     251                \$faults)) {next;}
    222252
    223253        $pendingMinusFaults = $pending - $faults;
    224         print GNUDAT "$stage $faults $processed, $pendingMinusFaults\n";
    225         my $sum = $faults + $processed + $pendingMinusFaults;
    226         if ($sum > $maxY) {$maxY = $sum;}
     254        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
    227255    }
    228256
    … …  
    232260    use FileHandle;
    233261    GP->autoflush(1);
    234 
    235     if ($maxY == 0) {$maxY = 1;}
    236     else {$maxY = $maxY*1.1;}
    237 
    238262
    239263    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    … …  
    243267        "set grid;" .
    244268        "set boxwidth;" .
    245         "set yrange [\"0\":\"$maxY\"];" .
    246269        "set style data histogram;" .
    247270        "set style histogram rowstacked;" .
    … …  
    249272        "set ylabel \"Exposures\";" .
    250273        "set boxwidth 0.75;" .
    251         "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" .
     274        "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" .
    252275        "\n";
    253276
    254277    close GP;
    255     unlink($inputFile);
    256278}
    257279
    … …  
    295317    my ($self, $timeDiff, $divX, $timeFormat) = @_;
    296318
     319    # if less than a few mins of data, show 1 minute tics
     320    if ($timeDiff < 300) {
     321        ${$timeFormat} = "%H:%M";
     322        ${$divX} = 60;
     323    }
     324    # if less than a few mins of data, show 5 minute tics
     325    elsif ($timeDiff < 1800) {
     326        ${$timeFormat} = "%H:%M";
     327        ${$divX} = 300;
     328    }
     329    # if less than an hour of data, show 15 minute tics
     330    elsif ($timeDiff < 3600) {
     331        ${$timeFormat} = "%H:%M";
     332        ${$divX} = 900;
     333    }
    297334    # if less than a couple of hour's data plotted, show 30 mins tics
    298     if ($timeDiff < 7200) {
     335    elsif ($timeDiff < 7200) {
    299336        ${$timeFormat} = "%H:%M";
    300337        ${$divX} = 1800;
    … …  
    376413
    377414    my $firstIn = 1;
    378     foreach my $stage (keys %$gnuplotFiles) {
     415    # loop through stages array so that they are ordered properly (not maintained in hash)
     416    foreach my $stage (@allStages) {
     417
     418        if(!$gnuplotFiles->{$stage}) {next;}
    379419        if (!$firstIn) {print GP ",";}
    380420
    … …  
    398438
    399439    # now delete temp dat files
    400     foreach my $stage (keys %$gnuplotFiles) {
    401 
    402         unlink($gnuplotFiles->{$stage});
    403     }
     440    foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
    404441}
    405442
    … …  
    416453    my ($minX, $maxX, $minY, $maxY, $timeDiff);
    417454
    418     my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
     455    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     456    $self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
    419457
    420458    my $timeFormat = undef;
    … …  
    440478        "set ylabel \"Available (TB)\";" .
    441479        "plot " .
    442         "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
     480        "'".$tmpFile->filename."' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
    443481
    444482    print GP "\n";
    445483    close GP;
    446     unlink($gnuplotFile);
    447484}
    448485
    … …  
    458495        my $outputFile = "$prefix/czarplot_hosts_space.png";
    459496    my $limit = 97.0;
    460     my $inputFile = $self->{_czarDb}->createHostsData($limit);
     497    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     498    $self->{_czarDb}->createHostsData($limit, $tmpFile);
    461499
    462500    my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();
    … …  
    477515        "set ylabel \"Space (TB)\";" .
    478516        "set xtic rotate by -90 scale 0;" .
    479         "plot '$inputFile' " .
     517        "plot '".$tmpFile->filename."' " .
    480518        "using 2:xtic(1) t \"Used\" lt 7," .
    481519        "'' using 3 t \"Over $limit% used\" lt 1," .
    … …  
    489527    close GP;
    490528}
     529
     530###########################################################################
     531#
     532# Generates 3D plot of magic mask fraction for provided exposure ID
     533#
     534###########################################################################
     535sub plotMagicMaskFractionForThisExposure {
     536    my ($self, $exp_id) = @_;
     537
     538    # set up file stuff
     539    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
     540        my $outputFile = "$prefix/czarplot_single_exp_magic_mask_fraction.png";
     541    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_single_exp_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => '.dat');
     542
     543    my $fracs;
     544    if (!$self->{_gpc1Db}->getMagicMaskFraction($exp_id, \$fracs)) {return 0;}
     545
     546    open (GNUDAT, ">".$tmpFile->filename);
     547
     548    print GNUDAT "0 0 0.0\n";
     549    foreach my $row ( @{$fracs} ) {
     550
     551        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
     552            print GNUDAT "$1 $2 @{$row}[1]\n";
     553            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
     554            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
     555        }
     556    }
     557    print GNUDAT "7 7 0.0\n";
     558
     559
     560    close(GNUDAT);
     561
     562
     563    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     564    use FileHandle;
     565    GP->autoflush(1);
     566
     567    my $datFile = $tmpFile->filename;
     568
     569    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
     570    print GP <<PLOT;
     571
     572    set term $self->{_outputFormat}
     573    set title "Magic mask fraction for exposure $exp_id"
     574        set grid
     575        set size square
     576        set xrange [-0.5:7.5] reverse
     577        set yrange [-0.5:7.5] 
     578        unset key
     579        set palette rgb 22,13,10
     580        set ylabel "Y"
     581        set xlabel "X"
     582        set cbrange [0:1]
     583        set datafile missing "NaN"
     584        plot "$datFile" using 1:2:3  with image
     585PLOT
     586
     587    close GP;
     588
     589    return 1;
     590}
     591
     592
     593###########################################################################
     594#
     595# Plots mask fraction histogram for exposures taken between these dates
     596#
     597###########################################################################
     598sub plotMagicMaskFractionHistogram {
     599    my ($self, $begin, $end) = @_;
     600
     601    # get exposures between these dates
     602    my $exp_ids = undef;
     603    if (!$self->{_gpc1Db}->getProcessedExposures($begin, $end, \$exp_ids)) {return 0;}
     604    my $totalExp = @{$exp_ids};
     605    if ($totalExp < 1) {return 0;}
     606
     607    # set up file stuff
     608    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
     609        my $histoOutputFile = "$prefix/czarplot_magic_mask_fraction_h.png";
     610    my $distOutputFile = "$prefix/czarplot_magic_mask_fraction_d.png";
     611    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     612
     613    # set up bins
     614    my %histogram;
     615    my @bins;
     616    my $interval = 0.05;
     617    my $min = 1.0;
     618    my $max = 1.0;
     619    for (my $n=0.0; $n<$max; $n += $interval){
     620
     621        push(@bins, $n);
     622        $histogram{$n} = 0;
     623    }
     624
     625    # get mask for each exposure, and bin
     626    my $mask = undef;
     627    my $expCount = 0;
     628    foreach my $row ( @{$exp_ids} ) {
     629
     630        $mask = $self->{_gpc1Db}->getAverageMagicMaskFraction(@{$row}[0]);
     631
     632        if (!$mask) {next;}
     633        $expCount++;
     634
     635        foreach my $bin (@bins) {
     636            if ($mask <= ($bin+$interval)) {
     637
     638                $histogram{$bin}++;
     639                last;
     640            }
     641        }
     642    }
     643
     644    # write data to GNUPLOT data file
     645    print "* Found mask values for $expCount exposures out of $totalExp\n";
     646    open (GNUDAT, ">".$tmpFile->filename);
     647    my $accum = 0;
     648    my $maxBin = 0;
     649    foreach my $bin (@bins) {
     650
     651        $accum += $histogram{$bin};
     652        if ($histogram{$bin} > $maxBin) {$maxBin = $histogram{$bin};}
     653        print GNUDAT "$bin $histogram{$bin} $accum\n";
     654    }
     655    close(GNUDAT);
     656
     657    # now plot
     658    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     659    use FileHandle;
     660    GP->autoflush(1);
     661
     662    $maxBin = $maxBin * 1.1;
     663    $accum = $accum * 1.1;
     664
     665    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
     666    print GP
     667        "set term $self->{_outputFormat};" .
     668        "set title \"Magic mask fraction for $expCount exposures between '$begin' and '$end'\";" .
     669        "set grid;" .
     670        "set boxwidth;" .
     671        "set style data histogram;" .
     672        "set yrange [0:$maxBin];" .
     673        "set style histogram rowstacked;" .
     674        "set style fill solid border -1;" .
     675        "set ylabel \"Number of Exposures\";" .
     676        "set xlabel \"Magic Mask Fraction\";" .
     677        "set boxwidth 0.75;" .
     678        "set xtic rotate by -90 scale 0;" .
     679        "plot '".$tmpFile->filename."' using 2:xtic(1) notitle lt 1;" .
     680        "\n";
     681
     682    close GP;
     683
     684
     685    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     686    use FileHandle;
     687    GP->autoflush(1);
     688
     689    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
     690    print GP
     691        "set term $self->{_outputFormat};" .
     692        "set title \"Cumulative distribution of magic streaks per image between '$begin' and '$end'\";" .
     693        "set yrange [0:$accum];" .
     694        "set key left top;" .
     695        "set grid xtics;" .
     696        "set xlabel \"Magic Mask Fraction\";" .
     697        "set ylabel \"Number of Exposures\";" .
     698        "set xtic rotate by -90 scale 0;" .
     699        "plot " .
     700        "'".$tmpFile->filename."' using 1:3 notitle with lines lt 2 lw 2\n";
     701
     702    print GP "\n";
     703    close GP;
     704
     705    return 1;
     706}
    4917071;
Note: See TracChangeset for help on using the changeset viewer.