Index: /branches/eam_branches/ipp-20101103/Nebulous/Build.PL =================================================================== --- /branches/eam_branches/ipp-20101103/Nebulous/Build.PL (revision 29905) +++ /branches/eam_branches/ipp-20101103/Nebulous/Build.PL (revision 29906) @@ -122,4 +122,5 @@ bin/neb-touch bin/neb-xattr + bin/whichnode )], )->create_build_script; Index: /branches/eam_branches/ipp-20101103/Nebulous/bin/whichnode =================================================================== --- /branches/eam_branches/ipp-20101103/Nebulous/bin/whichnode (revision 29906) +++ /branches/eam_branches/ipp-20101103/Nebulous/bin/whichnode (revision 29906) @@ -0,0 +1,82 @@ +#!/bin/env perl + +# This script is a temporary "solution" to allow a client to determine whether +# what volumes contain instances of a nebulous storage object regardless +# of whether or not the volume is currently available. +# Used by cleanup and update processing + + +use strict; +use warnings; +use DBI; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use Nebulous::Key qw(parse_neb_key); + +my $dbname = "nebulous"; +my $dbserver = "ippdb00"; +my $verbose; + +GetOptions( + 'verbose' => \$verbose, # Print to stdout +) or pod2usage( 2 ); + +pod2usage( -msg => "filename required", -exitval => 2 ) if ! @ARGV; + + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +my $query = "SELECT volume.name AS volname, uri, mountedvol.*" + . " FROM storage_object JOIN instance USING(so_id)" + . " JOIN volume using(vol_id)" + . " LEFT JOIN mountedvol USING(vol_id) WHERE ext_id = ?"; +my $stmt = $dbh->prepare($query); + +my $errors = 0; +foreach my $in_key (@ARGV) { + my $key_object = parse_neb_key($in_key); + + die "$in_key is not a valid nebulous path\n" if !$key_object; + + my $key = $key_object->path; + + print STDERR "$key\n" if $verbose; + + $stmt->execute($key); + my $num_instances = 0; + my $num_available = 0; + while ( my $instance= $stmt->fetchrow_hashref() ) { + my $volname = $instance->{volname}; + my $uri = $instance->{uri}; + $num_instances++; + if ($instance->{available}) { + $num_available++; + print "$volname available\n"; + } else { + print "$volname not available\n"; + $errors++; + } + } +} + +exit 0; + +sub getDBHandle { + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + Index: /branches/eam_branches/ipp-20101103/Nebulous/lib/Nebulous/Client.pm =================================================================== --- /branches/eam_branches/ipp-20101103/Nebulous/lib/Nebulous/Client.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/Nebulous/lib/Nebulous/Client.pm (revision 29906) @@ -379,5 +379,5 @@ { # volume - type => SCALAR, + type => SCALAR|UNDEF, optional => 1, }, @@ -401,8 +401,15 @@ } + # set the number of user copies to 1 to prevent replication + $self->setxattr($key, 'user.copies', 1, 'replace'); + if (defined $locations) { - my $instances = $self->find_instances($key); - if (scalar @$instances < 2) { - die "not enough instances"; + my $instances = $self->find_instances($key, undef, 'find them all'); + if (scalar @$instances == 1) { + # only one instance nothing to do + return 0; + } + if (scalar @$instances == 0) { + die "no instances"; } foreach my $victim (@$instances) { @@ -416,6 +423,10 @@ # start at one so cull() is called one less time then the # of # instances - if ($stats->[6] < 2) { - die "not enough instances"; + if ($stats->[6] == 1) { + # only one instance nothing to do + return 0; + } + if ($stats->[6] == 0) { + die "no instances"; } for (my $i = 1; $i < $stats->[6]; $i++) { @@ -439,4 +450,37 @@ } +sub storage_object_exists +{ + my $self = shift; + + # HI! + + my ($key) = validate_pos(@_, + { + type => SCALAR, + }, + ); + + $log->debug( "entered - @_" ); + + my $found = 0; + eval { + my $objects = $self->find_objects($key); + if ($objects) { + $found = (scalar @$objects) > 0; + } + }; + if ($@) { + if ($@ =~ qr/invalid key/) { + $self->set_err($@); + return; + } + $log->logdie($@); + } + + $log->debug("leaving"); + + return $found; +} sub lock Index: /branches/eam_branches/ipp-20101103/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/eam_branches/ipp-20101103/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 29906) @@ -1087,4 +1087,128 @@ } +sub prepare_output +{ + my $self = shift; # Configuration object + my $name = shift; # Name of the filerule + my $outroot = shift; # For replacing {OUTPUT} + my $component = shift; # For replacing {CHIP.NAME} and {CELL.NAME} + my $delete_existing = shift; # if file exists delete it + my $r_error = shift; # reference to error code + + die "prepare_output: invalid number of arguments" if !defined $r_error; + + $$r_error = 0; + + my $output = $self->filename($name, $outroot, $component); + if (!$output) { + # error message emitted by filename should be sufficient + $$r_error = $PS_EXIT_CONFIG_ERROR; + return undef; + } + + if (file_scheme($output) ne 'neb') { + # non-nebulous file we're done + if ($delete_existing) { + if (!$self->file_delete($output)) { + carp "failed to delete $output"; + $$r_error = $PS_EXIT_SYS_ERROR; + $output = undef; + } + } + return $output; + } + + my $neb = $self->nebulous; + if ($neb->storage_object_exists($output)) { + if ($delete_existing) { + # avoid dead instances by moving the file before deleting it. + # append current time to form new name + my $todelete; + eval { + # parse the key so that we can compute the new name + require Nebulous::Key; + }; + if ($@) { + carp "Can't find Nebulous::Key"; + $$r_error = $PS_EXIT_CONFIG_ERROR; + return undef; + } + eval { + # parse the key so that we can compute the new name + my $neb_key = Nebulous::Key::parse_neb_key($output); + die "parse_neb_key failed" if !$neb_key; + my $path = $neb_key->path; + die "neb_key has no path" if !$path; + my $ticks = time(); + $todelete = "ipp_trash/$path.$ticks"; + $neb->move($output, $todelete); + }; + if ($@) { + carp "nebulous move failed for $output"; + $output = undef; + $$r_error = $PS_EXIT_SYS_ERROR; + } + if ($todelete) { + eval { + $neb->delete($todelete); + }; + if ($@) { + carp "nebulous delete for $todelete failed. Ignoring.\n"; + $$r_error = $PS_EXIT_SYS_ERROR; + } + } + } else { + # Make sure that there is only 1 instance. + + eval { + $neb->there_can_be_only_one($output); + }; + if ($@) { + carp "nebulous there_can_be_only_one() failed for $output"; + $output = undef; + $$r_error = $PS_EXIT_SYS_ERROR; + } + } + } + return $output; +} + +# Cause a nebulous file to be replicated setting the user.copies value +sub replicate_file +{ + my $self = shift; + my $file = shift; + my $copies = shift; + + if (file_scheme($file) ne 'neb') { + carp "cannot replicate non-neulous file: $file"; + return 0; + } + + if (!$copies) { + $copies = 2; + } + + my $neb = $self->nebulous; + eval { + $neb->setxattr($file, 'user.copies', $copies, 'replace'); + }; + if ($@) { + carp "failed to set user.copies for $file"; + return 0; + } + + # XXX: if copies > 2 should we make more replicants here ? + eval { + $neb->replicate($file); + }; + if ($@) { + carp "failed to replicate $file"; + return 0; + } + return 1; +} + + # Return catdir for tessellation, from TESSELLATIONS within the site configuration sub tessellation_catdir Index: /branches/eam_branches/ipp-20101103/dbconfig/cam.md =================================================================== --- /branches/eam_branches/ipp-20101103/dbconfig/cam.md (revision 29905) +++ /branches/eam_branches/ipp-20101103/dbconfig/cam.md (revision 29906) @@ -113,4 +113,8 @@ maskfrac_max_magic F32 0.0 maskfrac_max_advisory F32 0.0 + deteff F32 0 + deteff_err F32 0 + deteff_lq F32 0 + deteff_uq F32 0 quality S16 0 END Index: /branches/eam_branches/ipp-20101103/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20101103/dbconfig/changes.txt (revision 29905) +++ /branches/eam_branches/ipp-20101103/dbconfig/changes.txt (revision 29906) @@ -1981,2 +1981,11 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + +ALTER TABLE chipProcessedImfile ADD column deteff_magref FLOAT; +ALTER TABLE camProcessedExp ADD column deteff FLOAT AFTER maskfrac_max_advisory; +ALTER TABLE camProcessedExp ADD column deteff_err FLOAT AFTER deteff; +ALTER TABLE camProcessedExp ADD column deteff_lq FLOAT AFTER deteff_err; +ALTER TABLE camProcessedExp ADD column deteff_uq FLOAT AFTER deteff_lq; +UPDATE dbversion set schema_version = '1.1.66', updated= CURRENT_TIMESTAMP(); + +-- Version 1.1.67 Index: /branches/eam_branches/ipp-20101103/dbconfig/chip.md =================================================================== --- /branches/eam_branches/ipp-20101103/dbconfig/chip.md (revision 29905) +++ /branches/eam_branches/ipp-20101103/dbconfig/chip.md (revision 29906) @@ -105,4 +105,5 @@ maskfrac_magic F32 0.0 maskfrac_advisory F32 0.0 + deteff_magref F32 0.0 END Index: /branches/eam_branches/ipp-20101103/dbconfig/config.md =================================================================== --- /branches/eam_branches/ipp-20101103/dbconfig/config.md (revision 29905) +++ /branches/eam_branches/ipp-20101103/dbconfig/config.md (revision 29906) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.65 + pkg_version STR 1.1.66 END Index: /branches/eam_branches/ipp-20101103/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/ipp-20101103/ippScripts/scripts/camera_exp.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippScripts/scripts/camera_exp.pl (revision 29906) @@ -70,9 +70,19 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $cam_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); - if (not defined $run_state) { $run_state = 'new'; } -if ($run_state eq 'update') { - $logDest .= '.update'; + +my_die ("$run_state is an invalid value for run-state", $cam_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update'); + + +my $replicateOutputs = 1; + +my $logDest; +my $traceDest; +if ($run_state eq 'new') { + $logDest = prepare_output("LOG.EXP", $outroot, undef, 0); + $traceDest = prepare_output("TRACE.EXP", $outroot, undef, 0); +} else { + $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, undef, 0); + $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, undef, 0); } @@ -188,5 +198,5 @@ print $list4File ($chipMask . "\n"); - push @outMasks, $ipprc->filename("PSASTRO.OUTPUT.MASK", $outroot, $class_id) if $produceMasks; + push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks; } close $list1File; @@ -199,24 +209,17 @@ # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs -my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $fpaStats = $ipprc->filename("PSASTRO.STATS", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); - -if ($run_state eq 'update') { - $traceDest .= '.update'; - $fpaStats .= '.update'; -} - -# convert supplied DVO database name to UNIX filename -my $dvodbReal; -if (defined $dvodb) { - $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO - $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); -} - -#my $dtime_addstar = 0; +my $jpeg1 = prepare_output("PPIMAGE.JPEG1", $outroot, undef, 1); +my $jpeg2 = prepare_output("PPIMAGE.JPEG2", $outroot, undef, 1); +my $fpaObjects = prepare_output("PSASTRO.OUTPUT", $outroot, undef, 1); +my $configuration = prepare_output("PSASTRO.CONFIG", $outroot, undef, 1); + +my $do_stats; +my $fpaStats; +if ($run_state eq 'new') { + $do_stats = 1; + $fpaStats = prepare_output("PSASTRO.STATS", $outroot, undef, 1); +} else { + $do_stats = 0; +} unless ($no_op) { @@ -236,5 +239,5 @@ &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); } - &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1); + check_output($jpeg1, $replicateOutputs); } @@ -251,5 +254,5 @@ &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); } - &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2); + check_output($jpeg2, $replicateOutputs); } @@ -265,9 +268,6 @@ $command .= " -dbname $dbname" if defined $dbname; - my $do_stats; if ($run_state eq 'new') { - $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS"; $command .= " -dumpconfig $configuration"; - $do_stats = 1; } elsif ($run_state eq 'update') { $command .= " -ipprc $configuration"; @@ -275,4 +275,5 @@ &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR); } + $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS" if $do_stats; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -286,6 +287,7 @@ my $quality; # Quality flag if ($do_stats) { + check_output($fpaStats, $replicateOutputs); + my $fpaStatsReal = $ipprc->file_resolve($fpaStats); - &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $fpaStatsReal ); # parse stats from metadata @@ -306,59 +308,14 @@ if (!$quality) { - &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($fpaObjects); + check_output($fpaObjects, $replicateOutputs); foreach my $outMask (@outMasks) { - &my_die("Unable to find expected output file: $outMask", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outMask); + check_output($outMask, $replicateOutputs); } if ($run_state eq 'new') { - &my_die("Couldn't find expected output file: $configuration", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); + check_output($configuration, $replicateOutputs); } } - - # run addstar on the output fpaObjects (if a DVO database is defined) -# if (defined $dvodbReal and ($run_state eq 'new')) { - -# ## XXX the camera analysis can either save the full set of -# ## detections, or just the image metadata, in the dvodb - -# ## get the addstar recipe for this camera and CAMERA reduction -# $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; -# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); -# } -# my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or -# &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); - -# ## allow the dvodb to save only images, or the full detection set -# my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - -# # XXX this construct requires the user to have a valid .ptolemyrc -# # XXX which in turn points at ippconfig/dvo.site -# # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) -# # XXX this needs to be converted to addstar_client... - -# my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar -# my $command; -# $command = "$addstar -D CAMERA $camdir -update"; -# $command .= " -image" if $imagesOnly; -# $command .= " -D CATDIR $dvodbReal"; - -# my $realFile = $ipprc->file_resolve($fpaObjects); -# $command .= " $realFile"; - -# my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); -# } -# $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script -# } } } @@ -374,5 +331,4 @@ $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= " -dtime_script $dtime_script"; -# $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { $fpaCommand .= " -updaterun -set_state full"; @@ -393,4 +349,37 @@ } +exit 0; + +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $class_id = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $cam_id, $error); + + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $cam_id, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists($file); + + if ($replicate and (file_scheme($file) eq 'neb')) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $cam_id, $PS_EXIT_SYS_ERROR); + } +} sub my_die Index: /branches/eam_branches/ipp-20101103/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20101103/ippScripts/scripts/chip_imfile.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippScripts/scripts/chip_imfile.pl (revision 29906) @@ -66,5 +66,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state", +pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state --dbname", -exitval => 3) unless defined $exp_id and @@ -75,12 +75,25 @@ defined $camera and defined $outroot and + defined $dbname and defined $run_state; my_die ("$run_state is an invalid value for run-state", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update'); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration - -my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -$logDest .= ".update" if $run_state eq "update"; +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my ($logDest, $traceDest); +if ($run_state eq 'new') { + $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0); + $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1); +} else { + $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1); + $traceDest = prepare_output("TRACE.IMFILE.UPDATE", $outroot, $class_id, 1); +} if ($redirect) { @@ -90,4 +103,5 @@ print STDOUT "FULL COMMAND: $0 @ARGS\n\n"; } + # Recipes to use based on reduction class @@ -114,16 +128,46 @@ ## these names are used in ppImage, and thus may be URIs -my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $outputWeight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); -my $configuration = $ipprc->filename("PPIMAGE.CONFIG", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); - -if ($run_state eq 'update') { - $outputStats .= '.update'; - $traceDest .= '.update'; +my $outputImage = prepare_output("PPIMAGE.CHIP", $outroot, $class_id, 1 ); +my $outputMask = prepare_output("PPIMAGE.CHIP.MASK", $outroot, $class_id, 1); +my $outputWeight = prepare_output("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id, 1); +my $pattern = prepare_output("PPIMAGE.PATTERN", $outroot, $class_id, 1); +my $backmdl = prepare_output("PSPHOT.BACKMDL", $outroot, $class_id, 1); + +my $configuration; +my $outputSources; +my $outputPsf; +my $outputStats; +my $outputBin1; +my $outputBin2; +my $dump_config = 1; +my $do_binned_images = 1; +if ($run_state eq 'new') { + # prepare the files that are only created for a new run + $configuration = prepare_output("PPIMAGE.CONFIG", $outroot, $class_id, 1); + $outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); +} else { + $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) + or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($configuration)) { + $dump_config = 0; + } else { + print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; + + # XXX: should we create a new config dump file? + # I vote yes but only if we can distingusing between temporarily unavailable and GONE. + my $gone = 0; + if (storage_object_exists($configuration, \$gone)) { + if ($gone) { + rename_gone_file($configuration); + $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1); + # if we dump the config we need to insure that the config dump represents + # the full processing + } else { + # file is temporarily not available. Don't dump config. + $dump_config = 0; + } + } + } + # make sure that any lingering destreak backup files are gone $ipprc->delete_destreak_backup_file($outputImage) @@ -134,4 +178,8 @@ or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); } +if ($do_binned_images) { + $outputBin1 = prepare_output("PPIMAGE.BIN1", $outroot, $class_id, 1); + $outputBin2 = prepare_output("PPIMAGE.BIN2", $outroot, $class_id, 1); +} my $cmdflags; @@ -152,4 +200,19 @@ my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + my $do_photom = metadataLookupBool($recipeData, 'PHOTOM'); + if ($do_photom and ($run_state eq 'update')) { + # If previous photometry outputs are ok skip running photometry + if ($dump_config || rerun_photometry($outroot, $class_id)) { + carp "Will rerun photometry\n"; + } else { + $do_photom = 0; + } + } + if ($do_photom) { + $outputSources = prepare_output("PSPHOT.OUTPUT", $outroot, $class_id, 1); + $outputPsf = prepare_output("PSPHOT.PSF.SAVE", $outroot, $class_id, 1); + } + ## XXX make the feature more general? @@ -357,29 +420,29 @@ } + $command = "$ppImage -file $uri $outroot"; + if ($dump_config) { + $command .= " -recipe PPIMAGE $recipe_ppImage"; + $command .= " -dumpconfig $configuration"; + } else { + $command .= " -ipprc $configuration"; + } + if ($do_photom) { + $command .= " -recipe PSPHOT $recipe_psphot"; + } else { + $command .= " -Db PPIMAGE:PHOTOM FALSE"; + } if ($run_state eq "new") { - $command = "$ppImage -file $uri $outroot"; - $command .= " -recipe PPIMAGE $recipe_ppImage"; - $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -threads $threads" if defined $threads; - $command .= " -dbname $dbname" if defined $dbname; - $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id; - $command .= " -source_id $source_id" if defined $source_id; - $command .= " -dumpconfig $configuration"; - $command .= " -tracedest $traceDest -log $logDest"; - $do_stats = 1; - } else { - $command = "$ppImage -file $uri $outroot"; - $command .= " -ipprc $configuration"; - $command .= " -threads $threads" if defined $threads; - $command .= " -dbname $dbname" if defined $dbname; - $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id; - $command .= " -source_id $source_id" if defined $source_id; - $command .= " -tracedest $traceDest -log $logDest"; - $command .= " -Db PPIMAGE:PHOTOM FALSE"; - } - if ($do_stats) { $command .= " -recipe PPSTATS CHIPSTATS"; $command .= " -stats $outputStats"; - } + $do_stats = 1; + } + if (!$do_binned_images) { + $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE"; + } + $command .= " -threads $threads" if defined $threads; + $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id; + $command .= " -source_id $source_id" if defined $source_id; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -394,4 +457,6 @@ my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS'); my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS'); + my $outputBackmdlExpect = metadataLookupBool($recipeData, 'BACKGROUND'); + my $outputPatternExpect = (metadataLookupBool($recipeData, 'PATTERN.ROW') or metadataLookupBool($recipeData, 'PATTERN.CELL')) ; my $quality; # Quality flag @@ -416,14 +481,21 @@ if (!$quality) { - &my_die("Couldn't find expected output file: $outputImage\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage); - &my_die("Couldn't find expected output file: $outputMask\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask); - &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight); - &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); - &my_die("Couldn't find expected output file: $outputBin2\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); - if ($run_state eq 'new') { - &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); - } - } - + my $replicateImages = 0; + check_output($outputImage, $replicateImages) if $outputImageExpect; + check_output($outputMask, $replicateImages) if $outputMaskExpect; + check_output($outputWeight, $replicateImages) if $outputWeightExpect; + if ($do_binned_images) { + check_output($outputBin1, 1); + check_output($outputBin2, 1); + } + check_output($configuration, 1) if $dump_config; + check_output($backmdl, 1) if $outputBackmdlExpect; + check_output($pattern, 1) if $outputPatternExpect; + if ($do_photom) { + check_output($outputSources, 1); + check_output($outputPsf, 1); + } + # XXX: Do we want to replicate the stats, logs, trace file? + } } @@ -462,4 +534,192 @@ } +exit 0; + +# check whether psphot outputs should be regenerated. +# Whether we need to or not is a somewhat complicated question. +sub rerun_photometry +{ + my $outroot = shift; + my $class_id = shift; + my $outputSources = $ipprc->filename('PSPHOT.OUTPUT', $outroot, $class_id) + or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + + my $make_sources = 0; + my $sources_available = 0; + if ($ipprc->file_exists($outputSources)) { + $sources_available = 1; + } else { + carp "WARNING: photometry sources file $outputSources is not available"; + my $gone; + if (storage_object_exists($outputSources, \$gone)) { + # check whether the file is permanantely or temporarily gone + if ($gone) { + carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + rename_gone_file($outputSources); + $make_sources = 1; + } + } else { + # storage object must have been deleted + $make_sources = 1; + } + } + + my $make_psf = 0; + my $psf_available = 0; + my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $class_id) + or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($outputPsf)) { + $psf_available = 1; + } else { + carp "PSF file $outputPsf is missing"; + my $gone = 0; + if (storage_object_exists($outputPsf, \$gone)) { + # object exists, but no instances are available. If they are permanently gone + # rename the storage object + if ($gone) { + carp "WARNING: PSF storage object exists but all instances are permanently gone"; + rename_gone_file($outputPsf); + $make_psf = 1; + } + } else { + $make_psf = 1; + } + } + + if ($sources_available && $psf_available) { + return 0; + } + + # if either of the files are gone rerun photometry unless the other file is temporarily not available + + if (!$sources_available && !$make_sources) { + # destreak will die if the sources is not available + &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + if (!$psf_available && !$make_psf) { + # warp updates need the psf file + &my_die("PSPHOT.PSF.SAVE is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + return $make_psf || $make_sources; +} + +# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that +# does not exist and one that all of the instances have been lost. +# XXXX This should be implemented properly in Nebulous +# For now uses Bill's script 'whichnode' which queries the nebulous database directly + +my $whichnode; +sub storage_object_exists +{ + my $file = shift; + my $ref_all_gone = shift; + + my $exists = $neb->storage_object_exists($file); + if (!$exists) { + return 0; + } + + if (!$whichnode) { + $whichnode = can_run('whichnode') or + &my_die("Can't find whichnode", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + } + + my $command = "$whichnode $file"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform whichnode: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + } + + my @lines = split "\n", (join "", @$stdout_buf); + + if (scalar @lines == 0) { + # no output the file is really and truely gone + # XXX: this is now caught above + print STDERR "storage object for $file does not exist\n"; + return 0; + } + + my $numGone = 0; + my $numNotGone = 0; + foreach my $line (@lines) { + chomp $line; + + # output lines are either + # "volume available" + # or + # "volume not available" + + my ($volume, $answer, undef) = split " ", $line; + # our hack is if the volume has an X in the name it's gone + if ($volume =~ /X/) { + print STDERR "$file is on $volume which is gone\n"; + $numGone++; + } elsif ($answer eq 'available') { + $numNotGone++; + } elsif ($answer eq 'not') { + print STDERR "$file is on $volume which is not available\n"; + $numNotGone++; + } else { + print STDERR "unexpected output from whichnode: $line\n"; + } + } + # if there are any instances that are not on a gone volume set all_gone to 0 + if ($numNotGone == 0 and $numGone > 0) { + $$ref_all_gone = 1; + } else { + $$ref_all_gone = 0; + } + + # storage object exists so return true + return 1; +} +sub rename_gone_file +{ + # check whether the only instance of file is on a lost volume + # XXX: we don't have a proper interface for this. + # For now try to use Bill's hack: the script 'whichnode' + + my $file = shift; + $neb->move($file, "$file.gone") or + &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $class_id = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $exp_id, $chip_id, $class_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + if ($replicate and (file_scheme($file) eq 'neb')) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } +} sub my_die Index: /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl (revision 29906) @@ -1984,5 +1984,5 @@ sub file_gone { - # if $check_for_gone check whether the only instance of file is on a lost volumen + # if $check_for_gone check whether the only instance of file is on a lost volume # XXX: we don't have a proper interface for this. # For now try to use Bill's hack the script 'whichnode' @@ -1990,4 +1990,8 @@ my $file = shift; + + if (file_scheme($file) ne 'neb') { + return 0; + } if (!$whichnode) { @@ -2021,4 +2025,6 @@ print STDERR "$file is on $volume which is gone\n"; $numGone++; + } elsif ($answer eq 'available') { + $numNotGone++; } elsif ($answer eq 'not') { print STDERR "$file is on $volume which is not available\n"; Index: /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_inject_fileset.pl =================================================================== --- /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_inject_fileset.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_inject_fileset.pl (revision 29906) @@ -24,6 +24,8 @@ # Parse the command-line arguments my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +my $exp_name; GetOptions('camera|i=s' => \$camera, # user-supplied camera name 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'exp_name|=s' => \$exp_name, # user-supplied exp_name name 'workdir|w=s' => \$workdir, # working directory for output files 'reduction=s' => \$reduction, # user-supplied camera name @@ -79,7 +81,6 @@ } -# use the first file name as the exp_name (strip off .fits) +# if not supplied, use the first file name as the exp_name (strip off .fits) my $num = 0; -my $exp_name; foreach my $file ( @ARGV ) { # check for file existence @@ -89,5 +90,5 @@ # strip off the extension my ( $vol, $path, $name ) = File::Spec->splitpath( $file ); - ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts|flt)(|.gz)/; print "exp_name : $exp_name.\n"; } Index: /branches/eam_branches/ipp-20101103/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/ipp-20101103/ippTasks/destreak.pro (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTasks/destreak.pro (revision 29906) @@ -305,5 +305,5 @@ task destreak.revert.load host local - periods -poll 60.0 + periods -poll 5 periods -exec 120. periods -timeout 120 Index: /branches/eam_branches/ipp-20101103/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/share/camtool_find_pendingimfile.sql (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/share/camtool_find_pendingimfile.sql (revision 29906) @@ -2,4 +2,7 @@ camRun.cam_id, chipProcessedImfile.*, + (IF(exp_time IS NOT NULL AND deteff_magref IS NOT NULL, + (2.5 * LOG10(exp_time) + chipProcessedImfile.deteff_magref), NULL)) + AS deteff_inst, rawExp.exp_name, rawExp.camera, Index: /branches/eam_branches/ipp-20101103/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29906) @@ -14,4 +14,4 @@ AND (chipRun.magicked = 0 OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') - AND (magicDSFile.data_state = 'cleaned`' OR magicDSFile.data_state = 'update')) + AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) ) Index: /branches/eam_branches/ipp-20101103/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/share/pxadmin_create_tables.sql (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/share/pxadmin_create_tables.sql (revision 29906) @@ -371,4 +371,5 @@ maskfrac_magic FLOAT, maskfrac_advisory FLOAT, + deteff_magref FLOAT, PRIMARY KEY(chip_id, exp_id, class_id), KEY(data_state), @@ -486,5 +487,4 @@ n_astrom INT, path_base VARCHAR(255), - quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT NOT NULL, software_ver VARCHAR(16), @@ -499,4 +499,9 @@ maskfrac_max_magic FLOAT, maskfrac_max_advisory FLOAT, + deteff FLOAT, + deteff_err FLOAT, + deteff_lq FLOAT, + deteff_uq FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, PRIMARY KEY(cam_id), KEY(fault), @@ -1050,4 +1055,5 @@ hostname VARCHAR(64), good_frac FLOAT, + mjd_obs DOUBLE, quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, Index: /branches/eam_branches/ipp-20101103/ippTools/share/warptool_towarped_labels.sql =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/share/warptool_towarped_labels.sql (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/share/warptool_towarped_labels.sql (revision 29906) @@ -12,4 +12,2 @@ group by warpRun.label ORDER BY priority DESC, warp_id - limit 100 - Index: /branches/eam_branches/ipp-20101103/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/src/camtool.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/src/camtool.c (revision 29906) @@ -519,4 +519,25 @@ PXOPT_LOOKUP_F32(maskfrac_max_magic, config->args, "-maskfrac_max_magic", false, false); PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false); + + // we store actual detection efficiency by adding in zpt_obs + PXOPT_LOOKUP_F32(deteff_inst, config->args, "-deteff_inst", false, false); + PXOPT_LOOKUP_F32(deteff_inst_lq, config->args, "-deteff_inst_lq", false, false); + PXOPT_LOOKUP_F32(deteff_inst_uq, config->args, "-deteff_inst_uq", false, false); + // error is dd + PXOPT_LOOKUP_F32(deteff_err, config->args, "-deteff_inst_err", false, false); + psF32 deteff = NAN; + psF32 deteff_uq = NAN; + psF32 deteff_lq = NAN; + if (isfinite(zpt_obs)) { + if (isfinite(deteff_inst)) { + deteff = deteff_inst + zpt_obs; + } + if (isfinite(deteff_inst_uq)) { + deteff_uq = deteff_inst_uq + zpt_obs; + } + if (isfinite(deteff_inst_lq)) { + deteff_lq = deteff_inst_lq + zpt_obs; + } + } /* psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", */ @@ -666,4 +687,8 @@ maskfrac_max_magic, maskfrac_max_advisory, + deteff, + deteff_err, + deteff_lq, + deteff_uq, quality ); Index: /branches/eam_branches/ipp-20101103/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/src/camtoolConfig.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/src/camtoolConfig.c (revision 29906) @@ -198,4 +198,9 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-maskfrac_max_advisory", 0, "define advisory mask fraction", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst", 0, "define deteff", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_err", 0, "define deteff_err", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_lq", 0, "define deteff_lq", NAN); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN); + // -processedexp Index: /branches/eam_branches/ipp-20101103/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/src/chiptool.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/src/chiptool.c (revision 29906) @@ -601,4 +601,5 @@ PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + PXOPT_LOOKUP_F32(deteff_magref, config->args, "-deteff_magref", false, false); psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", @@ -706,5 +707,6 @@ maskfrac_dynamic, maskfrac_magic, - maskfrac_advisory + maskfrac_advisory, + deteff_magref )) { // rollback @@ -715,20 +717,4 @@ return false; } - -#if 0 - // XXX I've decided to make the transaction cover the Exp migration as - // well. Otherwise, if the last imfile in an exp is moved and the exp - // migration fails then the data base is left in a situation where the exp - // migration can't happen. - - if (!chipProcessedCompleteExp(config)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } -#endif if (!psDBCommit(config->dbh)) { Index: /branches/eam_branches/ipp-20101103/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/src/chiptoolConfig.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/src/chiptoolConfig.c (revision 29906) @@ -196,4 +196,5 @@ psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + psMetadataAddF32(addprocessedimfileArgs, PS_LIST_TAIL, "-deteff_magref", 0, "define deteff_magref", NAN); // -processedimfile Index: /branches/eam_branches/ipp-20101103/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/ipp-20101103/ippTools/src/warptool.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippTools/src/warptool.c (revision 29906) @@ -994,5 +994,5 @@ psString thisWhere = NULL; if (whereStr) { - psStringAppend(&thisWhere, whereStr); + psStringAppend(&thisWhere, "\n%s", whereStr); } psStringAppend(&thisWhere, "\nAND warpRun.label = '%s'", label); Index: /branches/eam_branches/ipp-20101103/ippconfig/isp/camera.config =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/isp/camera.config (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/isp/camera.config (revision 29906) @@ -77,3 +77,3 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID} # Rule for generating photcode # don't censor any masked pixels when making postage stamps -MASK.NO.CENSOR U32 0 +MASK.NO.CENSOR U32 4294967295 Index: /branches/eam_branches/ipp-20101103/ippconfig/megacam/camera.config =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/megacam/camera.config (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/megacam/camera.config (revision 29906) @@ -155,3 +155,3 @@ # don't censor any masked pixels -NO.CENSOR U32 0xFFFFFFFF +MASK.NO.CENSOR U32 4294967295 Index: /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-mef.mdc (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-mef.mdc (revision 29906) @@ -319,7 +319,11 @@ LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +LOG.IMFILE.UPDATE OUTPUT {OUTPUT}.{CHIP.NAME}.log.update TEXT NONE CHIP TRUE NONE +LOG.EXP.UPDATE OUTPUT {OUTPUT}.log.update TEXT NONE FPA TRUE NONE TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE +TRACE.IMFILE.UPDATE OUTPUT {OUTPUT}.{CHIP.NAME}.trace.update TEXT NONE CHIP TRUE NONE +TRACE.EXP.UPDATE OUTPUT {OUTPUT}.trace.update TEXT NONE FPA TRUE NONE PPNOISEMAP.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE Index: /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-simple.mdc (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-simple.mdc (revision 29906) @@ -246,4 +246,8 @@ PPSUB.REF.CONV.VARIANCE OUTPUT {OUTPUT}.refConv.wt.fits VARIANCE NONE FPA TRUE NONE +PPSUB.FORCED.SOURCES OUTPUT {OUTPUT}.frc.cmf CMF NONE FPA TRUE NONE +PPSUB.POS1.SOURCES OUTPUT {OUTPUT}.pos1.cmf CMF NONE FPA TRUE NONE +PPSUB.POS2.SOURCES OUTPUT {OUTPUT}.pos2.cmf CMF NONE FPA TRUE NONE + PPSTACK.OUTPUT.COMP OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE PPSTACK.OUTPUT.RAW OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE @@ -290,7 +294,11 @@ LOG.IMFILE OUTPUT {OUTPUT}.imfile.log TEXT NONE FPA TRUE NONE LOG.EXP OUTPUT {OUTPUT}.exp.log TEXT NONE FPA TRUE NONE +LOG.IMFILE.UPDATE OUTPUT {OUTPUT}.imfile.log.update TEXT NONE FPA TRUE NONE +LOG.EXP.UPDATE OUTPUT {OUTPUT}.exp.log.update TEXT NONE FPA TRUE NONE TRACE.IMFILE OUTPUT {OUTPUT}.imfile.trace TEXT NONE FPA TRUE NONE TRACE.EXP OUTPUT {OUTPUT}.exp.trace TEXT NONE FPA TRUE NONE +TRACE.IMFILE.UPDATE OUTPUT {OUTPUT}.imfile.trace.update TEXT NONE FPA TRUE NONE +TRACE.EXP.UPDATE OUTPUT {OUTPUT}.exp.trace.update TEXT NONE FPA TRUE NONE PPNOISEMAP.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE Index: /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-split.mdc (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/recipes/filerules-split.mdc (revision 29906) @@ -323,7 +323,11 @@ LOG.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.log TEXT NONE CHIP TRUE NONE LOG.EXP OUTPUT {OUTPUT}.log TEXT NONE FPA TRUE NONE +LOG.IMFILE.UPDATE OUTPUT {OUTPUT}.{CHIP.NAME}.log.update TEXT NONE CHIP TRUE NONE +LOG.EXP.UPDATE OUTPUT {OUTPUT}.log.update TEXT NONE FPA TRUE NONE TRACE.IMFILE OUTPUT {OUTPUT}.{CHIP.NAME}.trace TEXT NONE CHIP TRUE NONE TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE +TRACE.IMFILE.UPDATE OUTPUT {OUTPUT}.{CHIP.NAME}.trace.update TEXT NONE CHIP TRUE NONE +TRACE.EXP.UPDATE OUTPUT {OUTPUT}.trace.update TEXT NONE FPA TRUE NONE PPNOISEMAP.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE Index: /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStats.config =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStats.config (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStats.config (revision 29906) @@ -84,4 +84,6 @@ HEADER STR IQ_M4_LQ HEADER STR IQ_M4_UQ + HEADER STR DETEFF.MAGREF + HEADER STR PSLIB_V Index: /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStatsFromMetadata.config (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/recipes/ppStatsFromMetadata.config (revision 29906) @@ -187,4 +187,7 @@ ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory + + # Detection Efficiency + ENTRY VAL DETEFF.MAGREF F32 CONSTANT -deteff_magref END @@ -255,4 +258,10 @@ ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage ENTRY VAL STREAK_V STR CONSTANT -ver_streaks + + # Detection Efficiency + ENTRY VAL deteff_inst F64 ROBUST_MEDIAN -deteff_inst + ENTRY VAL deteff_inst F64 ROBUST_STDEV -deteff_inst_err + ENTRY VAL deteff_inst F64 UQ -deteff_inst_uq + ENTRY VAL deteff_inst F64 LQ -deteff_inst_lq END Index: /branches/eam_branches/ipp-20101103/ippconfig/simtest/camera.config =================================================================== --- /branches/eam_branches/ipp-20101103/ippconfig/simtest/camera.config (revision 29905) +++ /branches/eam_branches/ipp-20101103/ippconfig/simtest/camera.config (revision 29906) @@ -80,4 +80,4 @@ # don't censor any masked pixels when making postage stamps -# MASK.NO.CENSOR U32 0xFFFFFFFF -MASK.NO.CENSOR U32 0 +# MASK.NO.CENSOR U32 0xFFFFFFFF -- need to fix the MDC parser(s) to handle hex values +MASK.NO.CENSOR U32 4294967295 Index: /branches/eam_branches/ipp-20101103/ppImage/src/ppImageDetrendRecord.c =================================================================== --- /branches/eam_branches/ipp-20101103/ppImage/src/ppImageDetrendRecord.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/ppImage/src/ppImageDetrendRecord.c (revision 29906) @@ -75,4 +75,5 @@ detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", "Fringe filename"); + detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename"); psFree (detrend); return true; Index: /branches/eam_branches/ipp-20101103/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/eam_branches/ipp-20101103/pstamp/scripts/pstamp_job_run.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/pstamp/scripts/pstamp_job_run.pl (revision 29906) @@ -119,4 +119,5 @@ $command .= " -dbserver $dbserver" if $dbserver; $command .= " -stage $params->{stage}" if $params->{stage}; + $command .= " -no_censor_masked" unless $nan_masked; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /branches/eam_branches/ipp-20101103/pstamp/src/ppstampArguments.c =================================================================== --- /branches/eam_branches/ipp-20101103/pstamp/src/ppstampArguments.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/pstamp/src/ppstampArguments.c (revision 29906) @@ -25,4 +25,5 @@ fprintf(stderr, " [-mask mk_image] : mask image\n"); fprintf(stderr, " [-variance var_image] : variance image\n"); + fprintf(stderr, " [-no_censor_masked] : do not set masked pixels to NAN\n"); fprintf(stderr, "\n"); @@ -95,4 +96,8 @@ } + if ((argnum = psArgumentGet(argc, argv, "-no_censor_masked"))) { + psArgumentRemove(argnum, &argc, argv); + options->censorMasked = false; + } // finally the output file Index: /branches/eam_branches/ipp-20101103/pstamp/src/ppstampMakeStamp.c =================================================================== --- /branches/eam_branches/ipp-20101103/pstamp/src/ppstampMakeStamp.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/pstamp/src/ppstampMakeStamp.c (revision 29906) @@ -327,5 +327,5 @@ } - if (!setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) { + if (options->censorMasked && !setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) { psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n"); status = false; Index: /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.c =================================================================== --- /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.c (revision 29905) +++ /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.c (revision 29906) @@ -28,4 +28,5 @@ options->roip.dDEC = 0; options->chipName = NULL; + options->censorMasked = true; return options; Index: /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.h =================================================================== --- /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.h (revision 29905) +++ /branches/eam_branches/ipp-20101103/pstamp/src/ppstampOptions.h (revision 29906) @@ -15,4 +15,5 @@ // psRegion roi; // roi in chip coordinates + bool censorMasked; } ppstampOptions; Index: /branches/eam_branches/ipp-20101103/tools/czarmetrics.pl =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czarmetrics.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czarmetrics.pl (revision 29906) @@ -9,4 +9,6 @@ use czartool::DayMetrics; +use czartool::MultiDayMetrics; +use czartool::MetricsIndex; use czartool::CzarDb; use czartool::Gpc1Db; @@ -16,4 +18,6 @@ my $begin = undef; my $end = undef; +my $cumulative = undef; +my $index = undef; my $verbose = undef; my $save_temps = undef; @@ -23,5 +27,7 @@ "begin|b=s" => \$begin, "end|e=s" => \$end, - "day|y=s" => \$day, + "day|d=s" => \$day, + "index|i" => \$index, + "cumulative|c" => \$cumulative, "verbose|v" => \$verbose, ); @@ -35,11 +41,17 @@ } if (!$day) { - print "* OPTIONAL: choose a single day -y (default=today)\n"; + print "* OPTIONAL: choose a single day -d (default=today)\n"; } -if (!$begin) { - print "* OPTIONAL: choose a begin date -b (default=today)\n";} -if (!$end) -{ - print "* OPTIONAL: choose an end date -e (default=today)\n"; +if (!$begin) { + print "* OPTIONAL: choose a begin date -b (default=today)\n"; +} +if (!$end) { + print "* OPTIONAL: choose an end date -e (default=today)\n"; +} +if (!$cumulative) { + print "* OPTIONAL: cumulative metrics for date range -c (default=off)\n"; +} +if (!$index) { + print "* OPTIONAL: create index file for all metrics -i (default=off)\n"; } @@ -52,4 +64,10 @@ $czarDb->setDateFormat("%Y%m%d-%H%i%s"); +if ($index) { + + my $metricsIndex = new czartool::MetricsIndex($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0); + $metricsIndex->writeHTML(); + exit; +} if (!$day && !$begin && !$end) { @@ -61,10 +79,20 @@ my $thisDay = $begin; -while (1) { - my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay); - $dayMetrics->writeHTML(); +if ($cumulative) { - $thisDay = $czarDb->addInterval($thisDay, "1 DAY"); - if ($czarDb->isBefore($end, $thisDay)) {last;} + my $multiDayMetrics = new czartool::MultiDayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $begin, $end); + $multiDayMetrics->writeHTML(); } +else { + + while (1) { + + my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay); + $dayMetrics->writeHTML(); + + $thisDay = $czarDb->addInterval($thisDay, "1 DAY"); + if ($czarDb->isBefore($end, $thisDay)) {last;} + } +} + Index: /branches/eam_branches/ipp-20101103/tools/czarplot.pl =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czarplot.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czarplot.pl (revision 29906) @@ -45,5 +45,5 @@ "begin|b=s" => \$begin, "end|e=s" => \$end, - "day|y=s" => \$day, + "day|d=s" => \$day, "output|o=s" => \$path, "histogram|h" => \$histogram, @@ -51,5 +51,5 @@ "cleanup|c" => \$showCleanup, "rate|r" => \$rate, - "deriv|d" => \$deriv, + "deriv|f" => \$deriv, "analysis|a" => \$analysis, "timeseries|t" => \$timeSeries, @@ -80,5 +80,5 @@ if (!$deriv) { $deriv = 0; - print "* OPTIONAL: plot first derivative -d (default=$deriv)\n";} + print "* OPTIONAL: plot first derivative -f (default=$deriv)\n";} if (!$analysis) { $analysis = 0; @@ -106,5 +106,5 @@ print "* OPTIONAL: choose an end time -e (default=now)\n";} if (!$day) { - print "* OPTIONAL: choose a single day to plot -y (default=today)\n";} + print "* OPTIONAL: choose a single day to plot -d (default=today)\n";} if (!$path) { print "* OPTIONAL: choose an output location -o (default=none)\n"; @@ -137,9 +137,15 @@ if($day) { - # day plots should run from about 6:30am until midnight - $begin = "$day 06:35"; - $end = "$day 23:59"; - print "JKJKJK '$begin' '$end' \n"; + if ($magicMask) { + $begin = $day; + $end = $day; + } + else { + + # day plots should run from about 6:30am until midnight + $begin = "$day 06:35"; + $end = "$day 23:59"; + } } else { @@ -152,8 +158,7 @@ } } -my $diskUsage = 1; # TODO +if ($rate) { -if ($rate) { - $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval, $log); + $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval); exit; } @@ -165,5 +170,5 @@ if ($exposureId) {$plotter->plotMagicMaskFractionForThisExposure($exposureId);} - else {$plotter->plotMagicMaskFractionHistogram($begin, $end);} + else {$plotter->plotMagicMaskFraction($begin, $end);} } Index: /branches/eam_branches/ipp-20101103/tools/czarpoll.pl =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czarpoll.pl (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czarpoll.pl (revision 29906) @@ -133,8 +133,8 @@ print "* Creating metrics for last 24 hours\n"; - # TODO hardcopded path needs to be in config - my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $today); + my $yesterday = $czarDb->subtractInterval($today, "1 DAY"); + # TODO hardcoded path needs to be in config + my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $yesterday); $dayMetrics->writeHTML(); - $doneMetricsToday = 1; } Index: /branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm (revision 29906) @@ -719,8 +719,13 @@ SQL - $query->execute; + if (!$query->execute) {return 0;} + (${$pending}, ${$faults}) = $query->fetchrow_array(); ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime); + + if (!${$pending}) {${$pending} = 0;} + if (!${$faults}) {${$faults} = 0;} + if (!${$processed}) {${$processed} = 0;} return 1; @@ -822,9 +827,10 @@ SQL - if (!$query->execute) {return undef;} (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); + + if (!${$maxY} || !${$minY} || !${$maxX} || !${$minX} || !${$timeDiff}) {return 0;} $query = $self->{_db}->prepare(<new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); - $tmpFile->unlink_on_destroy( 0 ); + $tmpFile->unlink_on_destroy(0); ${$dataFile} = $tmpFile->filename; @@ -873,10 +881,9 @@ while(1) { - - #print "NNN $startTime, $end\n"; if (!$self->isBefore($startTime, $end)) {last;} $endTime = $self->addInterval($startTime, $interval); - $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults); + if (!$self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults)) {} $timestamp = $self->getFormattedDate($endTime); + if (!$processed) {$processed = 0;} print GNUDAT "$timestamp $processed 0 0\n"; @@ -887,6 +894,4 @@ close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"; - - #return $someData; } Index: /branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czartool/DayMetrics.pm (revision 29906) @@ -43,8 +43,8 @@ # create path, dir and html file - $self->{dayDir} = "$self->{baseDir}/$self->{day}"; - rmdir($self->{dayDir}); - mkdir($self->{dayDir}, 0777); - $self->{plotter}->setOutputPath($self->{dayDir}); + $self->{path} = "$self->{baseDir}/$self->{day}"; + rmdir($self->{path}); + mkdir($self->{path}, 0777); + $self->{plotter}->setOutputPath($self->{path}); if ($self->{verbose}) {print "* Creating metrics for $self->{day}\n";} @@ -64,10 +64,8 @@ # create HTML file and write header stuff - open (HTMLDOC, ">$self->{dayDir}/index.html"); - print HTMLDOC "\n"; - print HTMLDOC "\n"; - print HTMLDOC "IPP Metrics for $self->{day}\n"; - print HTMLDOC "\n"; - print HTMLDOC "

IPP Metrics for $self->{day}

\n"; + + + $self->createHtml("IPP Metrics for $self->{day}"); + my $htmlFile = $self->{htmlFile}; # summit and burntool exposures @@ -79,11 +77,11 @@ my $nextDay = $self->{czarDb}->addInterval($self->{day}, "1 DAY"); - print HTMLDOC "
"; - print HTMLDOC " \< previous day | next day \>
\n"; - print HTMLDOC "Measured from $self->{begin} to $self->{end} HST
\n"; - printf ( HTMLDOC "%d exposures taken on summit last night, %d through burntool today
\n", + print $htmlFile "
"; + print $htmlFile " \< previous day | all | next day \>
\n"; + print $htmlFile "Measured from $self->{begin} to $self->{end} HST
\n"; + printf ( $htmlFile "%d exposures taken on summit last night, %d through burntool today
\n", $summitExposures, $burntoolMetrics->getProcessed() ? $burntoolMetrics->getProcessed() : 0 ); - print HTMLDOC "\n"; - print HTMLDOC "\n"; + print $htmlFile "\n"; + print $htmlFile "\n"; @@ -176,31 +174,27 @@ } - # create hyperlink list of list of active labels - print HTMLDOC "

Active labels for this day

\n"; + # create contents + print $htmlFile "

Contents

\n"; + print $htmlFile "Survey timings
\n"; + print $htmlFile "Summary plots
\n"; + print $htmlFile "Stage totals
\n"; + print $htmlFile "Magic mask fraction
\n"; foreach my $label (keys (%labelTables)) { - print HTMLDOC "$label
\n"; - } - - # some plots - print HTMLDOC "

Summary plots

\n"; - $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0); - print HTMLDOC "\"All{day}\" />\n"; - $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0); - print HTMLDOC "\"All{day}\" />\n"; - $self->{plotter}->createRateTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, "1 HOUR", 0); - print HTMLDOC "\"All{day}\" />\n"; - $self->{plotter}->plotMagicMaskFractionHistogram($self->{day}, $self->{day}); - print HTMLDOC "\"\"\n"; - $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end}); - print HTMLDOC "\"All{day}\" />\n"; - print HTMLDOC "\"\"\n"; - + print $htmlFile "$label
\n"; + } # survey table $self->printSurveyTable(); + # some summary plots + $self->createSummaryPlots(); + + # magic mask plots + $self->createMaskPlots($self->{day}, $self->{day}); + # table for stage totals - $table = "

Totals for all labels

\n"; + $table = "\n"; + $table .= "

Stage totals (top)

\n"; $table .= ""; $table .= " \n"; @@ -224,24 +218,21 @@ $table .= "
\n"; - print HTMLDOC $table; + print $htmlFile $table; # print all label tables to page foreach my $label (keys (%labelTables)) { - print HTMLDOC "\n"; - print HTMLDOC "

$label (top)

\n"; + print $htmlFile "\n"; + print $htmlFile "

$label (top)

\n"; $self->{plotter}->createTimeSeries($label, undef, $self->{begin}, $self->{end}, 0, 0, 0); - print HTMLDOC "\"All{day}\" />\n"; + print $htmlFile "\"All{day}\" />\n"; $self->{plotter}->createHistogram($label, $self->{begin}, $self->{end}, 0, 0, 0); - print HTMLDOC "\"All{day}\" />\n"; - - print HTMLDOC $labelTables{$label}; - } - - - print HTMLDOC "
\n"; - print HTMLDOC "\n"; - print HTMLDOC "\n"; - close(HTMLDOC); + print $htmlFile "\"All{day}\" />\n"; + + print $htmlFile $labelTables{$label}; + } + + + $self->finishHtml(); } @@ -254,12 +245,15 @@ my ($self) = @_; - print HTMLDOC "

Survey statistics

\n"; - print HTMLDOC "
"; - print HTMLDOC " \n"; - print HTMLDOC " \n"; - print HTMLDOC " \n"; - print HTMLDOC " \n"; - print HTMLDOC " \n"; - print HTMLDOC " \n"; + my $htmlFile = $self->{htmlFile}; + + print $htmlFile "\n"; + print $htmlFile "

Survey timings (top)

\n"; + print $htmlFile "
SurveyStarted burntoolFinished distributionTime taken
"; + print $htmlFile " \n"; + print $htmlFile " \n"; + print $htmlFile " \n"; + print $htmlFile " \n"; + print $htmlFile " \n"; + print $htmlFile " \n"; # OSS survey @@ -307,5 +301,5 @@ $self->printSurveyDetails("All", $started, $finished, $timeTaken); - print HTMLDOC "
SurveyStarted burntoolFinished distributionTime taken
\n"; + print $htmlFile "\n"; } @@ -318,10 +312,12 @@ my ($self, $survey, $started, $finished, $timeTaken, $processed, $pending) = @_; - print HTMLDOC " \n"; - print HTMLDOC " $survey\n"; - printf (HTMLDOC " %s\n", $started ? $started : "no"); - printf (HTMLDOC " %s\n", $finished ? $finished : "no"); - printf (HTMLDOC " %s\n", $timeTaken ? $timeTaken : "na"); - print HTMLDOC " \n"; + my $htmlFile = $self->{htmlFile}; + + print $htmlFile " \n"; + print $htmlFile " $survey\n"; + printf ($htmlFile " %s\n", $started ? $started : "no"); + printf ($htmlFile " %s\n", $finished ? $finished : "no"); + printf ($htmlFile " %s\n", $timeTaken ? $timeTaken : "na"); + print $htmlFile " \n"; } 1; Index: /branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm (revision 29906) @@ -246,12 +246,13 @@ ########################################################################### # -# Returns average magic mask fraction across all chips for a particular exposure -# -########################################################################### -sub getAverageMagicMaskFraction { - my ($self, $exp_id) = @_; +# Returns average magic mask fraction, sum of mask fractions and chip count +# for a particular exposure +# +########################################################################### +sub getMagicMaskStats { + my ($self, $exp_id, $mean, $sum, $chipCount) = @_; my $query = $self->{_db}->prepare(<execute; - return scalar $query->fetchrow_array(); + if (!$query->execute) {return 0;} # TODO do this everywhere + + (${$mean}, ${$sum}, ${$chipCount}) = $query->fetchrow_array(); + + return 1; } 1; Index: /branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czartool/Metrics.pm (revision 29906) @@ -47,3 +47,77 @@ } +########################################################################### +# +# Writes HTML intro +# +########################################################################### +sub createHtml { + my ($self, $title) = @_; + + open (FILE, ">$self->{path}/index.html"); + $self->{htmlFile} = *FILE; + + my $htmlFile = $self->{htmlFile}; + print $htmlFile "\n"; + print $htmlFile "\n"; + print $htmlFile "$title\n"; + print $htmlFile "\n"; + print $htmlFile "

$title

\n"; +} + +########################################################################### +# +# Creates summary plots for period +# +########################################################################### +sub createSummaryPlots { + my ($self) = @_; + + my $htmlFile = $self->{htmlFile}; + + print $htmlFile "\n"; + print $htmlFile "

Summary plots (top)

\n"; + $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0); + print $htmlFile "\"timeseries\"\n"; + $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0); + print $htmlFile "\"histogram\"\n"; + $self->{plotter}->createRateTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef, 0); + print $htmlFile "\"rate\"\n"; + $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end}); + print $htmlFile "\"storage\n"; +} + +########################################################################### +# +# Creates magic mask plots for provided period +# +########################################################################### +sub createMaskPlots { + my ($self, $begin, $end) = @_; + + my $htmlFile = $self->{htmlFile}; + + print $htmlFile "\n"; + print $htmlFile "

Magic mask fraction (top)

\n"; + + $self->{plotter}->plotMagicMaskFraction($begin, $end); + print $htmlFile "\"\"\n"; + print $htmlFile "\"\"\n"; +} + +########################################################################### +# +# Writes HTML intro +# +########################################################################### +sub finishHtml { + my ($self) = @_; + + my $htmlFile = $self->{htmlFile}; + + print $htmlFile "
\n"; + print $htmlFile "\n"; + print $htmlFile "\n"; + close($htmlFile); +} 1; Index: /branches/eam_branches/ipp-20101103/tools/czartool/MetricsIndex.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/MetricsIndex.pm (revision 29906) +++ /branches/eam_branches/ipp-20101103/tools/czartool/MetricsIndex.pm (revision 29906) @@ -0,0 +1,113 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use POSIX qw/strftime/; +use CGI::Pretty qw[:standard]; + +use czartool::CzarDb; +use czartool::Gpc1Db; +use czartool::Plotter; +use czartool::StageMetrics; +use czartool::Metrics; + + +package czartool::MetricsIndex; +our @ISA = qw(czartool::Metrics); # inherits from Metrics class + +########################################################################### +# +# Constructor (overrides superclass) +# +########################################################################### +sub new { + my ($class) = @_; + + # Call the constructor of the parent class + my $self = $class->SUPER::new( + $_[1], # gpc1Db + $_[2], # czarDb + $_[3], # baseDir + $_[4], # verbose + $_[5] # save_temps + ); + + # create path, dir and html file + $self->{path} = "$self->{baseDir}"; + + if ($self->{verbose}) {print "* Creating index file for metrics\n";} + + bless $self, $class; + + return $self; +} + +########################################################################### +# +# Writes HTML document +# +########################################################################### +sub writeHTML { + my ($self) = @_; + + $self->createHtml("IPP Metrics"); + my $htmlFile = $self->{htmlFile}; + + opendir(DIR, $self->{path}) or die $!; + + # read metrics base dir and store contents in array + my @days; + my @lunations; + while (my $dir = readdir(DIR)) { + + next if ($dir =~ m/^\./); + next if (-f "$self->{path}/$dir"); + + if ($dir =~ m/[0-9]{4}-[0-9]{2}-[0-9]{2}_.*/) {push(@lunations, $dir);} + else { push(@days, $dir);} + + } + closedir(DIR); + + # sort array then write to HTML + print $htmlFile "

Lunations

\n"; + @lunations = sort{$b cmp $a}(@lunations); + foreach my $lunation (@lunations) { + + print "$lunation\n"; + print $htmlFile "$lunation
\n"; + } + + # sort array then write to HTML + my @months = qw( NA Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec ); + @days = sort{$b cmp $a}(@days); + my $lastMonth = "NA"; + my $lastYear = "NA"; + foreach my $day (@days) { + + if ($day =~ m/^([0-9]{4})-([0-9]{2})-[0-9]{2}$/) { + + # print year, if changes + if ($1 ne $lastYear) { + + print $htmlFile "

$1

"; + $lastYear = $1; + } + # print month, if changes + if ($2 ne $lastMonth) { + + print $htmlFile "

".$months[$2]."

"; + $lastMonth = $2; + } + print "$day\n"; + print $htmlFile "$day
\n"; + } + } + + $self->finishHtml(); + + return 1; +} +1; Index: /branches/eam_branches/ipp-20101103/tools/czartool/MultiDayMetrics.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/MultiDayMetrics.pm (revision 29906) +++ /branches/eam_branches/ipp-20101103/tools/czartool/MultiDayMetrics.pm (revision 29906) @@ -0,0 +1,78 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use POSIX qw/strftime/; +use CGI::Pretty qw[:standard]; + +use czartool::CzarDb; +use czartool::Gpc1Db; +use czartool::Plotter; +use czartool::StageMetrics; +use czartool::Metrics; + + +package czartool::MultiDayMetrics; +our @ISA = qw(czartool::Metrics); # inherits from Metrics class + +########################################################################### +# +# Constructor (overrides superclass) +# +########################################################################### +sub new { + my ($class) = @_; + + # Call the constructor of the parent class + my $self = $class->SUPER::new( + $_[1], # gpc1Db + $_[2], # czarDb + $_[3], # baseDir + $_[4], # verbose + $_[5] # save_temps + ); + + $self->{startDay} = $_[6]; + $self->{endDay} = $_[7]; + + # sort out times + $self->{begin} = "$self->{startDay} 06:00"; + $self->{end} = "$self->{endDay} 06:00"; + + # create path, dir and html file + $self->{path} = "$self->{baseDir}/$self->{startDay}_$self->{endDay}"; + rmdir($self->{path}); + mkdir($self->{path}, 0777); + $self->{plotter}->setOutputPath($self->{path}); + + if ($self->{verbose}) {print "* Creating metrics for $self->{startDay} to $self->{endDay}\n";} + + bless $self, $class; + + return $self; +} + +########################################################################### +# +# Writes HTML document +# +########################################################################### +sub writeHTML { + my ($self) = @_; + + $self->createHtml("IPP Metrics for $self->{startDay} to $self->{endDay}"); + my $htmlFile = $self->{htmlFile}; + + print $htmlFile "
all
"; + + # summary plots + $self->createSummaryPlots(); + + # magic mask plots + $self->createMaskPlots($self->{startDay}, $self->{endDay}); + + $self->finishHtml(); +} +1; Index: anches/eam_branches/ipp-20101103/tools/czartool/OCMetrics.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/OCMetrics.pm (revision 29905) +++ (revision ) @@ -1,93 +1,0 @@ - - -use DateTime; -use IPC::Cmd 0.36 qw( can_run run); -my $end = '2010-11-01'; -my $start = $czarDb->subtractInterval($end, "28 DAY"); - - -my $days; -getDaysFromFullMoon($start, \$days); -print "$start is $days from full moon\n"; -$start = $czarDb->addInterval($start, "$days DAY"); -print "=$start\n"; -getDaysFromFullMoon($end, \$days); -print "$end is $days from full moon\n"; -$end = $czarDb->addInterval($end, "$days DAY"); -print "=$end\n"; - - - -my ($beginl,$endl) = get_lunation_extent("2010-11-09"); - - -print "begin=$beginl end=$endl\n"; -exit; - - - - -########################################################################### -# -# Calls 'moondata' program to get number of days since/to full moon for provided date -# -########################################################################### -sub getDaysFromFullMoon() { - my ($date, $days) = @_; - - my $cmd = "moondata $date 0 0"; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run ( command => $cmd, verbose => $verbose); - unless ($success) {print "* Can't run '$cmd'\n"; return 0;} - - my @result = split /\s+/,(join "\n", @$stdout_buf); - ${$days} = $result[6]; - - return 1; -} - -sub get_lunation_extent { - my $date = shift; - my ($year,$month,$day) = split /-/,$date; - my $dateobs_begin; - my $dateobs_end; - - my $dt = DateTime->new(year => $year, month => $month, day => $day, - hour => 0, minute => 0, second => 0, nanosecond => 0, - time_zone => 'Pacific/Honolulu'); - do { - $dt->subtract(days => 1); - my $ymd = $dt->ymd; - - - my $md_cmd = "moondata $ymd 0 0"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $md_cmd, verbose => $verbose); - unless ($success) { - print "can;t run '$md_cmd'\n" - } - my @result = split /\s+/,(join "\n", @$stdout_buf); - if (abs($result[6]) <= 0.5) { - $dateobs_end = $ymd; - } - } while (!defined($dateobs_end)); - - do { - $dt->subtract(days => 1); - my $ymd = $dt->ymd; - my $md_cmd = "moondata $ymd 0 0"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $md_cmd, verbose => $verbose); - unless ($success) { - print "can;t run '$md_cmd'\n" - } - my @result = split /\s+/,(join "", @$stdout_buf); - if (abs($result[6]) <= 0.5) { - $dateobs_begin = $ymd; - } - } while (!defined($dateobs_begin)); - - return($dateobs_begin,$dateobs_end); -} - - Index: /branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm =================================================================== --- /branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm (revision 29905) +++ /branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm (revision 29906) @@ -1,3 +1,4 @@ #!/usr/bin/perl -w + package czartool::Plotter; @@ -56,5 +57,4 @@ } - ########################################################################### # @@ -67,10 +67,9 @@ my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin); - if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;} elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";} elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";} - elsif ($self->{_czarDb}->isBefore($timeDiff, "24:00:00")) {${$interval} = "1 HOUR";} # under 1 day + elsif ($self->{_czarDb}->isBefore($timeDiff, "26:00:00")) {${$interval} = "1 HOUR";} # under 1 day elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days @@ -83,4 +82,6 @@ else {${$interval} = "1 MONTH";} +# ${$interval} = "2 HOUR"; + return 1; } @@ -92,5 +93,5 @@ ########################################################################### sub createRateTimeSeries { - my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_; + my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_; # stages @@ -109,5 +110,5 @@ my $stage = undef; my $gnuplotFile = undef; - my $outputFile = createImageFileName($self, $label, $selectedStage, "r", $isLog); + my $outputFile = createImageFileName($self, $label, $selectedStage, "r", 0); open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; use FileHandle; @@ -118,5 +119,5 @@ print GP "set term $self->{_outputFormat};" . - "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" . + "set title \"'$label', '$selectedStage'\\nFrom '$beginTime' to '$endTime' HST\";" . "set boxwidth;" . "set xtic rotate by -90 scale 0;" . @@ -137,6 +138,5 @@ $endTime, $interval, - \$gnuplotFile, - $isLog)) { + \$gnuplotFile)) { $gnuplotFiles{$stage} = $gnuplotFile; @@ -251,5 +251,5 @@ \$faults)) {next;} - $pendingMinusFaults = $pending - $faults; + $pendingMinusFaults = $pending - $faults; print GNUDAT "$stage $faults $processed $pendingMinusFaults\n"; } @@ -264,5 +264,5 @@ print GP "set term $self->{_outputFormat};" . - "set title \"'$label', '$beginTime' to '$endTime'\";" . + "set title \"'$label'\\nFrom '$beginTime' to '$endTime' HST\";" . "set grid;" . "set boxwidth;" . @@ -379,5 +379,5 @@ my $yTitle = undef; if ($isLog) {$yTitle = "Log( Exposures )";} - elsif ($isDeriv) {$yTitle = "dExposures/dTime";} + elsif ($isDeriv) {$yTitle = "dExposures/dTime(secs)";} else {$yTitle = "Exposures";} @@ -392,5 +392,5 @@ else {$title .= "'all stages'"} - $title .= " for '$label', '$fromTime' to '$toTime'"; + $title .= " for '$label'\\nFrom '$fromTime' to '$toTime' HST"; open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; @@ -454,5 +454,5 @@ my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat'); - $self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff); + if (!$self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff)) {return 0;} my $timeFormat = undef; @@ -468,5 +468,5 @@ print GP "set term $self->{_outputFormat};" . - "set title \"Total available cluster space over time\";" . + "set title \"Total available cluster space over time\\nFrom $fromTime to $toTime HST\";" . "set key left top;" . "set xdata time;" . @@ -482,4 +482,6 @@ print GP "\n"; close GP; + + return 1; } @@ -530,5 +532,5 @@ ########################################################################### # -# Generates 3D plot of magic mask fraction for provided exposure ID +# Generates 2D plot of magic mask fraction for provided exposure ID # ########################################################################### @@ -546,4 +548,5 @@ open (GNUDAT, ">".$tmpFile->filename); + # missing corner chip print GNUDAT "0 0 0.0\n"; foreach my $row ( @{$fracs} ) { @@ -551,8 +554,10 @@ if (@{$row}[0] =~ m/XY([0-9])([0-9])/) { print GNUDAT "$1 $2 @{$row}[1]\n"; + # missing corner chips if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";} if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";} } } + # missing corner chip print GNUDAT "7 7 0.0\n"; @@ -596,5 +601,5 @@ # ########################################################################### -sub plotMagicMaskFractionHistogram { +sub plotMagicMaskFraction { my ($self, $begin, $end) = @_; @@ -623,16 +628,28 @@ } + my $meanMask = undef; + my $sumMask = undef; + my $chipCount = undef; + my $expCount = 0; + my $totalChipCount = 0; + my $totalMask = 0; + my $exp_id = undef; + # get mask for each exposure, and bin - my $mask = undef; - my $expCount = 0; foreach my $row ( @{$exp_ids} ) { - $mask = $self->{_gpc1Db}->getAverageMagicMaskFraction(@{$row}[0]); - - if (!$mask) {next;} + $exp_id = @{$row}[0]; + + if (!$self->{_gpc1Db}->getMagicMaskStats($exp_id, \$meanMask, \$sumMask, \$chipCount )) {next;} + + + if (!$meanMask) {next;} + #print "expId=$exp_id meanMask=$meanMask sumMask=$sumMask nChips=$chipCount\n"; $expCount++; + $totalMask += $sumMask; + $totalChipCount += $chipCount; foreach my $bin (@bins) { - if ($mask <= ($bin+$interval)) { + if ($meanMask <= ($bin+$interval)) { $histogram{$bin}++; @@ -640,4 +657,15 @@ } } + } + + my $totalMaskFrac; + + if ($totalChipCount > 0) { + + $totalMaskFrac = sprintf( "%.1f", ($totalMask/$totalChipCount) * 100.0); + } + else { + + $totalMaskFrac = "NA"; } @@ -655,16 +683,18 @@ close(GNUDAT); - # now plot + + $maxBin = $maxBin * 1.1; + $accum = $accum * 1.1; + + my $title = "Magic Mask Fraction for $expCount exposures\\nData taken between '$begin' and '$end'\\nTotal masked fraction is $totalMaskFrac%"; + + # make histogram open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; use FileHandle; GP->autoflush(1); - - $maxBin = $maxBin * 1.1; - $accum = $accum * 1.1; - if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";} print GP "set term $self->{_outputFormat};" . - "set title \"Magic mask fraction for $expCount exposures between '$begin' and '$end'\";" . + "set title \"$title\";" . "set grid;" . "set boxwidth;" . @@ -682,13 +712,12 @@ close GP; - + # make cumulative distribution open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; use FileHandle; GP->autoflush(1); - if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";} print GP "set term $self->{_outputFormat};" . - "set title \"Cumulative distribution of magic streaks per image between '$begin' and '$end'\";" . + "set title \"Cumulative distribution of $title\";" . "set yrange [0:$accum];" . "set key left top;" .