Index: /branches/eam_branches/ipp-20100823/Nebulous-Server/bin/neb-countcheck =================================================================== --- /branches/eam_branches/ipp-20100823/Nebulous-Server/bin/neb-countcheck (revision 29124) +++ /branches/eam_branches/ipp-20100823/Nebulous-Server/bin/neb-countcheck (revision 29124) @@ -0,0 +1,254 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005-2008 Joshua Hoblitt +# +# $Id: neb-admin,v 1.14 2008-10-16 22:24:39 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DBI; +use Net::Server::Daemonize qw( check_pid_file create_pid_file unlink_pid_file ); +use URI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ( + $db, + $dbhost, + $dbpass, + $dbuser, + $so_id_start, + $so_id_range, + $limit, + $verbose, +); + +$db = $ENV{'NEB_DB'}; +$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; +$dbuser = $ENV{'NEB_USER'}; +$dbpass = $ENV{'NEB_PASS'}; + +GetOptions( + 'db|d=s' => \$db, + 'host=s' => \$dbhost, + 'user|u=s' => \$dbuser, + 'pass|p=s' => \$dbpass, + 'so_id_start=i' => \$so_id_start, + 'so_id_range=i' => \$so_id_range, + 'limit|l=i' => \$limit, + 'verbose|v' => \$verbose, +) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) + unless $db && $dbuser && $dbpass; + +# set default limit to 5 +#$limit ||= 5; + +# check to make sure that only one instance of neb-admin is running +# XXX this implies we should move pending replicate elsewhere +# my $pidfile = '/var/tmp/neb-admin'; +# # abort if an instance is already running +# END { unlink_pid_file($pidfile) }; +# check_pid_file($pidfile); +# create_pid_file($pidfile); + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 1, + }, +); + +scan(); +# if ($pending) { +# pending(); +# } elsif ($removal) { +# removal(); +# } else { +# die "THIS SHOULD NOT HAPPEN"; +# } + +sub scan { + if (not defined $so_id_start) { $so_id_start = 0; } + if (not defined $so_id_range) { $so_id_range = 100000; } + my $so_id_end = $so_id_start + $so_id_range; + + # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10 + { + my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object"); + $query->execute; + my $answer = $query->fetchrow_arrayref; + my $max_so_id = $$answer[0]; + $query->finish; + + if ($so_id_start > $max_so_id) { + print STDERR "at end of so_id range, reset please\n"; + exit 10; + } + } + + my $sth = "SELECT so_id,Nhave,Nwant,ext_id FROM + (SELECT so_id,count(ins_id) AS Nhave,value AS Nwant FROM + instance + JOIN storage_object_xattr USING(so_id) + JOIN mountedvol USING(vol_id) + WHERE storage_object_xattr.name = 'user.copies' + AND mountedvol.available = 1 + AND so_id >= $so_id_start + AND so_id < $so_id_end + GROUP BY so_id + HAVING Nhave != Nwant + ) AS V + JOIN storage_object USING(so_id)"; + if ($limit) { + $sth .= " LIMIT $limit "; + } + + + my $query = $dbh->prepare($sth); + $query->execute; + + my @rows; + while (my $row = $query->fetchrow_hashref) { + my $call; + if ($row->{Nhave} > $row->{Nwant}) { + $call = 'neb-cull'; + } + elsif ($row->{Nhave} < $row->{Nwant}) { + $call = 'neb-replicate'; + } + else { + $call = 'neb-stat'; + } + print "$row->{so_id} $row->{Nhave} $row->{Nwant} $call $row->{ext_id}\n"; + push @rows, $row; + } + $query->finish; + + exit unless scalar @rows; +} +__END__ + +=pod + +=head1 NAME + +neb-countcheck + +=head1 SYNOPSIS + + neb-countcheck [--host ] [--db ] [--user ] + [--pass ] [--limit ] [--so_id_start ] [--so_id_range ] + +=head1 DESCRIPTION + +=head1 OPTIONS + +=over 4 + +=item * --limit|-l + +Limits the number of items return in response to some request (like +C<--pendingreplicate>). + +Optional. Defaults to 5. --limit 0 is treated as asking for the default +value. + +=item * --host + +Name of host on which the database resides. + +Optional. Defaults to C. + +=item * --db|-d + +Name of database (C) to create tables in. + +Optional if the appropriate environment variable is set. + +=item * --user|-u + +Username to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --pass|-p + +Password to authenticate with. + +Optional if the appropriate environment variable is set. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --db|-d + +=item * C + +Equivalent to --host + +=item * C + +Equivalent to --user|-u + +=item * C + +Equivalent to --pass|-p + +=back + +=head1 CREDITS + + +=head1 SUPPORT + + +=head1 AUTHOR + +CZW + +=head1 COPYRIGHT + +Copyright (C) 2010 Chris Waters. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L + +=cut Index: /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/ImageSelection.c =================================================================== --- /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/ImageSelection.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/ImageSelection.c (revision 29124) @@ -47,9 +47,16 @@ Image *MatchImage (unsigned int time, short int source, unsigned int imageID) { - int m; + int m = -1; if ((imageID != 0) && (imageID < Nimage)) { - m = (int) imageID - 1; - } else { + // imageID is in range for the array of images. If the table is still in order and + // no images have been deleted the index of the image we are looking for will be imageID - 1 + // If this is the case, we have it. Otherwise we'll have to go search for it below + int guess = (int) imageID - 1; + if (image[guess].imageID == imageID) { + m = guess; + } + } + if (m == -1) { m = match_image_subset (image, subset, Nsubset, time, source); } Index: /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/find_matches.c =================================================================== --- /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/find_matches.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/Ohana/src/opihi/dvo/find_matches.c (revision 29124) @@ -10,7 +10,7 @@ off_t *N1, *N2; double *X1, *Y1, *X2, *Y2; - double dX, dY, dR; + double dX, dY, dR, Doff, Dmin, Dmax, Rmin, Rmax; int status; - double RADIUS2, Rmin; + double RADIUS2, RadMin; off_t Nave; Coords tcoords; @@ -52,11 +52,27 @@ strcpy (tcoords.ctype, "RA---ARC"); + // this region includes a boundary layer of size RADIUS + if (fabs(region[0].Dmin) < fabs(region[0].Dmax)) { + Doff = RAD_DEG*region[0].Dmax; + } else { + Doff = RAD_DEG*region[0].Dmin; + } + if (Doff < 80) { + Rmin = region[0].Rmin - RADIUS / 3600.0 / cos(Doff); + Rmax = region[0].Rmax + RADIUS / 3600.0 / cos(Doff); + } else { + Rmin = 0.0; + Rmax = 360.0; + } + Dmin = region[0].Dmin - RADIUS / 3600.0; + Dmax = region[0].Dmax + RADIUS / 3600.0; + // identify the entries contained by this catalog & init index for (i = 0; i < Npoints; i++) { index[i] = -1; - if (RAvec->elements.Flt[i] < region[0].Rmin) continue; - if (RAvec->elements.Flt[i] > region[0].Rmax) continue; - if (DECvec->elements.Flt[i] < region[0].Dmin) continue; - if (DECvec->elements.Flt[i] > region[0].Dmax) continue; + if (RAvec->elements.Flt[i] < Rmin) continue; + if (RAvec->elements.Flt[i] > Rmax) continue; + if (DECvec->elements.Flt[i] < Dmin) continue; + if (DECvec->elements.Flt[i] > Dmax) continue; index[i] = -2; } @@ -100,5 +116,5 @@ /* within match range; look for matches */ Jmin = -1; - Rmin = RADIUS2; + RadMin = RADIUS2; for (J = j; (dX > -1.02*RADIUS) && (J < Nave); J++) { /* find closest match for this detection */ @@ -107,6 +123,6 @@ dR = dX*dX + dY*dY; if (dR > RADIUS2) continue; - if (dR < Rmin) { - Rmin = dR; + if (dR < RadMin) { + RadMin = dR; Jmin = J; } Index: /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 29123) +++ /branches/eam_branches/ipp-20100823/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 29124) @@ -1130,5 +1130,5 @@ } $cam_path_base = $run->{cam_path_base}; - $astrom_file = $run->{astrom_file}; + $astrom_file = $run->{astrom}; } my $astrom_file_resolved = $ipprc->file_resolve($astrom_file); Index: /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippMonitor/Makefile.in (revision 29124) @@ -61,5 +61,6 @@ $(DESTWWW)/cleanTmpDirectory.php \ $(DESTWWW)/warpProcessedExp_Images.php \ -$(DESTWWW)/warpProcessedExp.php +$(DESTWWW)/warpProcessedExp.php \ +$(DESTWWW)/diskUsage.php DEFSRC = \ Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/czartool_labels.php (revision 29124) @@ -39,5 +39,7 @@ if ($selectedStage == "") { $selectedStage = "all_stages"; } -echo 'Some documentation can be found here
'; +$nsStatus = getNightlyScienceStatus($czardb); +echo "

Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses).
Documentation can be found here and cluster load monitored here
Current nightly science status: $nsStatus

"; + // deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later @@ -66,5 +68,5 @@ $states=array("full","new","drop","wait"); -$stages=array("chip","cam","fake","warp","stack","diff","magic","magicDS","dist"); +$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist"); $servers=array( "addstar", @@ -87,5 +89,4 @@ echo "\n"; echo "\n"; -echo "

Current status of IPP (any faults are shown in parentheses). NOTE: This data is good as of: $lastUpdateTime

"; echo "\n"; @@ -231,5 +232,5 @@ echo "\n"; - echo "

Current labels for $server server

"; + echo "

Current labels for $server server (D=distributing, P=publishing)

"; write_header_cell($class, ""); @@ -238,4 +239,5 @@ foreach ($stages as &$stage) { + if ($stage == "burntool") continue; $reverting = getRevertStatus($db, $stage); $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode="; @@ -249,7 +251,6 @@ echo "\n"; echo "\n"; + write_header_cell($class, " "); write_header_cell($class, "Label (in order of priority)"); - write_header_cell($class, "Distributing?"); - write_header_cell($class, "Publishing?"); foreach ($stages as &$stage) { @@ -287,10 +288,17 @@ echo "\n"; + $distPub = " "; + if ($distributing) $distPub = "D"; + if ($publishing) $distPub = $distPub . "P"; + write_table_cell($class, '%s', "", $distPub); + write_table_cell($class, '%s', $link, $thisLabel); - write_table_cell($class, '%s', "", $distributing ? "yes" : "NO"); - write_table_cell($class, '%s', "", $publishing ? "yes" : "NO"); $str = ""; $anyFaults = false; + + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "burntool", $str, $anyFaults); + write_table_cell($class, '%s', $anyFaults ? $link : "", $str); $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel . "&chipRun.state=new"; @@ -337,4 +345,5 @@ echo "\n"; + write_table_cell($class, '%s', "", " "); write_table_cell($class, '%s', $link, "All $server labels"); @@ -355,4 +364,21 @@ $qry = $db->query($sql); if (dberror($qry)) {echo "error with $sql
\n";} +} + +########################################################################### +# +# Returns current nightly science status +# +########################################################################### +function getNightlyScienceStatus($db) { + + $sql = "SELECT status FROM nightlyscience"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + return $row[0]; } Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/diskUsage.php =================================================================== --- /branches/eam_branches/ipp-20100823/ippMonitor/raw/diskUsage.php (revision 29124) +++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/diskUsage.php (revision 29124) @@ -0,0 +1,206 @@ +$sql\n"; + $unitSize = 0.5; + } + + if ("$TABLE" == "distChipClean") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'chip' and $FILTER group by distRun.state"; + $unitSize = 0.5; + } + + if ("$TABLE" == "distWarpClean") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'warp' and $FILTER group by distRun.state"; + $unitSize = 0.5; + } + + if ("$TABLE" == "distDiffClean") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'diff' and $FILTER group by distRun.state"; + $unitSize = 0.5; + } + + $FILTER = "distRun.clean = 0 and distRun.state != 'cleaned'"; + if ("$TABLE" == "distRawFull") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'raw' and $FILTER group by distRun.state"; + $unitSize = 50; + } + + if ("$TABLE" == "distChipFull") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'chip' and $FILTER group by distRun.state"; + $unitSize = 60; + } + + if ("$TABLE" == "distWarpFull") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'warp' and $FILTER group by distRun.state"; + $unitSize = 80; + } + + if ("$TABLE" == "distDiffFull") { + $sql = "SELECT count(dist_id) as n, distRun.state from distRun join distComponent using (dist_id) where distRun.stage = 'diff' and $FILTER group by distRun.state"; + $unitSize = 80; + } + + $qry = $db->query($sql); + if (dberror($qry)) { + echo "\n"; + echo "\n"; + } + + $class = "list"; + $nlines = 0; + + $TbytesSum = 0; + + // list the results + while ($qry->fetchInto($row)) { + echo "\n"; + // ** TABLE DATA ** + + $Tbytes = $row[0]*$unitSize/1000000.0; + $TbytesSum += $Tbytes; + $TbytesAll += $Tbytes; + + write_table_cell ($class, '%s', "", $row[0]); + write_table_cell ($class, '%s', "", $row[1]); + write_table_cell ($class, '%s', "", $Tbytes); + write_table_cell ($class, '%s', "", $TbytesSum); + write_table_cell ($class, '%s', "", $TbytesAll); + echo "\n"; + $nlines ++; + } + echo "\n"; + + if ($nlines == 0) { + echo "\n"; + } +} + +$ID = checkID (); + +// require an explicit project +if (! $ID['proj']) { projectform ($ID); } + +$db = dbconnect($ID['proj']); + +if ($ID['menu']) { + $myMenu = $ID['menu']; +} else { + $myMenu = "ipp.imfiles.dat"; +} + +menu($myMenu, 'Disk Usage', 'ipp.css', $ID['link'], $ID['proj']); + +echo "

Night Summary

\n"; + +// set up the form +echo "\n"; + +$restricted = 0; + +// define restrictions to the queries +// ** TABLE RESTRICTIONS ** +$WHERE = check_restrict ('state', $WHERE, 'string', 1.0); + +// ensure no valid results +if ($restricted == 0) { + $WHERE = "$WHERE WHERE (0 > 1)"; +} + +$WHERE = check_ordering ('state,label', $WHERE); + +// set up the table +echo "
error reading $TABLE table

table query : $sql
$TABLE
------
$TABLEno match
\n"; + +// echo "\n"; // first field is a label set below for the query rows only +// ** TABLE HEADER ** +echo "\n"; +write_header_cell ("list", "N items"); +write_header_cell ("list", "state"); +write_header_cell ("list", "Tbytes"); +write_header_cell ("list", "Sum(TB)"); +write_header_cell ("list", "All(TB)"); +echo "\n"; + +// query restriction form +// echo "\n"; +// echo "\n"; +// ** TABLE QUERY ** (restictions) +echo "\n"; +write_query_row ('N items', 15, 'string'); +write_query_row ('state', 10, 'string'); +write_query_row ('Mbytes', 7, 'string'); +write_query_row ('Sum(MB)', 7, 'string'); +echo "\n"; + +// query the database (limit is 10, but should be 1 +table_section($db, $WHERE, "rawExp"); +table_section($db, $WHERE, "chipRun"); +table_section($db, $WHERE, "warpRun"); +table_section($db, $WHERE, "diffRun"); + +table_section($db, $WHERE, "distRawClean"); +table_section($db, $WHERE, "distChipClean"); +table_section($db, $WHERE, "distWarpClean"); +table_section($db, $WHERE, "distDiffClean"); + +table_section($db, $WHERE, "distRawFull"); +table_section($db, $WHERE, "distChipFull"); +table_section($db, $WHERE, "distWarpFull"); +table_section($db, $WHERE, "distDiffFull"); + +// close the table and form +echo "
Dataset
\n"; +echo "\n"; +$pass = $ID['pass']; +$proj = $ID['proj']; +$menu = $ID['menu']; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +// ** TAIL CODE ** + +echo " WHERE: $WHERE

\n"; +echo " SQL: $sql

\n"; + +menu_end(); + +?> Index: /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippMonitor/raw/ipp.imfiles.dat (revision 29124) @@ -38,2 +38,3 @@ menulink | menuselect | link | Tables columns | columns_in_db.php menulink | menuselect | link | Clean /tmp directory | cleanTmpDirectory.php +menutop | menutop | link | disk usage | diskUsage.php Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/automate_stacks.pl (revision 29124) @@ -123,5 +123,6 @@ defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or - defined $test_mode or defined $clean_old or defined $check_mode; + defined $test_mode or defined $clean_old or defined $check_mode or + defined $confirm_stacks; # Configurable parameters from our config file. @@ -1271,11 +1272,17 @@ print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n" } + if ($metadata_out{nsState} eq 'CONFIRM_STACKING') { + $metadata_out{nsState} = 'STACKING'; + } + next; } - if (defined($pretend)) { - add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target); - } - else { - add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target); + if ($metadata_out{nsState} eq 'CONFIRM_STACKING') { + if (defined($pretend)) { + add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target); + } + else { + add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target); + } } } Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_bundle.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_bundle.pl (revision 29124) @@ -154,5 +154,5 @@ # 2. magic is not required for this distRun # 3. the processing for the component produced no images (warp or diff with bad quality for example) -my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality); +my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack') || ($stage eq 'sky')) || $no_magic || $poor_quality); my ($image, $mask, $variance); Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_defineruns.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_defineruns.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/dist_defineruns.pl (revision 29124) @@ -83,5 +83,5 @@ push @stages, $stage; } else { - @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff); + @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff sky); } Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak_cleanup.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_destreak_cleanup.pl (revision 29124) @@ -38,5 +38,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera); +my ($magic_ds_id, $camera, $stage); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -44,4 +44,5 @@ 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier 'camera=s' => \$camera, # camera for evaluating file rules + 'stage=s' => \$stage, # camera for evaluating file rules 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -56,7 +57,6 @@ -exitval => 3) unless defined $magic_ds_id and - defined $camera; -# defined $stage and -# defined $stage_id; + defined $camera and + defined $stage; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -79,7 +79,14 @@ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n"; -my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, camRun.reduction AS cam_reduction, magicRun.inverse" - . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN camRun USING(cam_id)" - . " WHERE magic_ds_id = $magic_ds_id"; +my $q1; + +if ($stage ne 'diff') { + $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, camRun.reduction AS cam_reduction" + . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN camRun USING(cam_id)"; +} else { + $q1 = "SELECT magicDSRun.*, diffRun.diff_mode FROM magicDSRun JOIN diffRun ON stage_id = diffRun.diff_id AND stage = 'diff'"; +} +$q1 .= " WHERE magic_ds_id = $magic_ds_id"; + my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id"; @@ -89,12 +96,13 @@ &my_die ("Unable to find magicDSRun $magic_ds_id", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nrows; my $run = $stmt1->fetchrow_hashref(); +$stmt1->finish(); my $state = $run->{state}; -my $stage = $run->{stage}; my $stage_id = $run->{stage_id}; my $cam_path_base = $run->{cam_path_base}; my $cam_reduction = $run->{cam_reduction}; $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL'); -my $inverse = $run->{inverse}; + +my $warp_warp = ($stage eq 'diff' and $run->{diff_mode} eq 1); @@ -105,6 +113,9 @@ &my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; -my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use -&my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; +my $recipe_psastro; +if ($stage eq 'chip') { + $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use + &my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; +} @@ -217,5 +228,5 @@ delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); - if ($stage eq "diff" and $inverse) { + if ($stage eq "diff" and $warp_warp) { my $name = "PPSUB.INVERSE"; if ($backup_path_base) { Index: /branches/eam_branches/ipp-20100823/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippScripts/scripts/staticsky.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippScripts/scripts/staticsky.pl (revision 29124) @@ -71,4 +71,5 @@ $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect; +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); # my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_STATICSKY); @@ -164,5 +165,5 @@ $command .= " -recipe PSPHOT $recipe_psphot"; $command .= " -dumpconfig $configuration"; - # $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -tracedest $traceDest -log $logDest"; # $command .= " -dbname $dbname" if defined $dbname; # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; Index: /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTasks/destreak.pro (revision 29124) @@ -600,4 +600,5 @@ book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID book getword magicDSToCleanup $pageName camera -var CAMERA + book getword magicDSToCleanup $pageName stage -var STAGE book getword magicDSToCleanup $pageName outroot -var OUTROOT book getword magicDSToCleanup $pageName dbname -var DBNAME @@ -607,5 +608,5 @@ host anyhost - $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --logfile $logfile + $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --stage $STAGE --camera $CAMERA --logfile $logfile add_standard_args run Index: /branches/eam_branches/ipp-20100823/ippTasks/dist.pro =================================================================== --- /branches/eam_branches/ipp-20100823/ippTasks/dist.pro (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTasks/dist.pro (revision 29124) @@ -30,4 +30,5 @@ list DIST_STAGE -add "chip_bg" list DIST_STAGE -add "warp_bg" +list DIST_STAGE -add "sky" $currentStage = 0 Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl (revision 29124) @@ -10,4 +10,5 @@ use ippToPsps::IppToPspsDb; use ippToPsps::Datastore; +use ippToPsps::BatchManager; my $singleBatch = undef; @@ -17,4 +18,5 @@ my $toTime = undef; my $product = undef; +my $filePath = undef; @@ -24,4 +26,5 @@ 'to|t=s' => \$toTime, 'product|p=s' => \$product, + 'location|l=s' => \$filePath, 'verbose|v' => \$verbose, 'save_temps|s' => \$save_temps @@ -32,13 +35,16 @@ } if (!defined $singleBatch) { - print "* OPTIONAL: a single batch -b (default = none)\n"; + print "* OPTIONAL: a single batch -b (default = none)\n"; } if (!defined $fromTime) { $fromTime = "2010-01-01"; - print "* OPTIONAL: from time -f (default = $fromTime)\n"; + print "* OPTIONAL: from time -f (default = $fromTime)\n"; } if (!defined $toTime) { $toTime = "2099-12-31"; - print "* OPTIONAL: to time -t (default = $toTime)\n"; + print "* OPTIONAL: to time -t (default = $toTime)\n"; +} +if (!defined $filePath) { + print "* OPTIONAL: location for files to be deleted -l \n"; } if (!defined $verbose) { @@ -54,4 +60,5 @@ my $datastore = new ippToPsps::Datastore($product, 0, 0); my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); +my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $filePath, $verbose, $save_temps); my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus"; my $ua = LWP::UserAgent->new; @@ -72,6 +79,6 @@ my $numOfBatches; - if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getSingleBatch($singleBatch, \$batches);} - else { $numOfBatches = $ippToPspsDb->getBatchList(\$batches, $fromTime, $toTime);} + if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);} + else { $numOfBatches = $ippToPspsDb->getBatches(\$batches, $fromTime, $toTime);} if ($numOfBatches < 1) {return 0;} @@ -84,31 +91,70 @@ # loop round batches my $batch; + my $numBatchesToCheck = 0; my $numChecked = 0; + my $numRemovedFromDatastore = 0; + my $numDeleted = 0; foreach $batch ( @{$batches} ) { - my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @{$batch}; - - if (checkBatch($timestamp, $expId, $batchId, $surveyType, $deleted)) {$numChecked++;} - + my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) = @{$batch}; + + if (!$processed) {next;} + + $numBatchesToCheck++; + + my $batchName = $batchManager->getBatchName($batchId); + + # if not merged then update by polling ODM for status + if (!$merged) { + + if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;} + } + + # delete from datastore + if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) { + + $deleted = $datastore->remove($batchName); + if ($deleted) { + $ippToPspsDb->setBatchAsDeleted($batchId, $expId); + $numRemovedFromDatastore++; + } + } + # if merged and already removed from datastore then delete files + if (defined $filePath && $merged && $deleted) { + + if($batchManager->deleteBatch($batchId)) {$numDeleted++;} + } + + # update database + $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted); + printf( "| %18s | %11s | %10d | %6s | %6s | %6s | %5s |\n", + $timestamp, + $batchName, + $expId, + $loadedToOdm ? "yes" : "no", + $mergeWorthy ? "yes" : "no", + $merged ? "yes" : "no", + $deleted ? "yes" : "no"); } + printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n"); - printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numOfBatches); - -} - -######################################################################################y + printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck); + printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es"); + printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es"); + +} + +######################################################################################## # # Check a single batch # ######################################################################################## -sub checkBatch { - my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @_; - - - my $batchFilter = sprintf("B%08d", $batchId); +sub checkODM { + my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_; + my $statusFilter = "*"; my $response = $ua->post($odmUrl, - [batchNameFilter => $batchFilter, + [batchNameFilter => "%".$batchName, statusFilter => $statusFilter, fromFilter => "2010-01-01", @@ -119,6 +165,6 @@ if ($response->code != 200) { - print "Problem connecting to web service for '$batchFilter', HTTP response status: ".$response->status_line."\n"; - return; + print "Problem connecting to web service for '$batchName', HTTP response status: ".$response->status_line."\n"; + return 0; } # print( "HTTP response status: ".$response->content."\n" ); @@ -128,33 +174,11 @@ close($tempFile); - my $loadedToOdm = 0; - my $mergeWorthy = 0; - my $mergeCompleted = 0; - - parseXml($tempName, \$loadedToOdm, \$mergeWorthy, \$mergeCompleted); - - # delete from datastore - if(defined $product) { - - if (!$deleted && $loadedToOdm && $mergeWorthy) { - - $deleted = $datastore->remove($batchFilter); - if ($deleted) { - $ippToPspsDb->setBatchAsDeleted($batchId, $expId); - } - } - } - - printf( "| %18s | %11s | %10d | %6s | %6s | %6s | %5s |\n", - $timestamp, - $batchFilter, - $expId, - $loadedToOdm ? "yes" : "no", - $mergeWorthy ? "yes" : "no", - $mergeCompleted ? "yes" : "no", - $deleted ? "yes" : "no"); - - $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted, $deleted); - + ${$loadedToOdm} = 0; + ${$mergeWorthy} = 0; + ${$merged} = 0; + + parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged); + + return 1; } @@ -165,5 +189,5 @@ ######################################################################################## sub parseXml { - my ($xmlFile, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_; + my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_; my $parser = XML::LibXML->new; @@ -174,9 +198,9 @@ ${$loadedToOdm} = 0; ${$mergeWorthy} = 0; - ${$mergeCompleted} = 0; + ${$merged} = 0; if ($result =~ m/LoadStarted/) { ${$loadedToOdm} = 1;} if ($result =~ m/MergeWorthy/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1;} - if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$mergeCompleted} = 1;} -} - + if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$merged} = 1;} +} + Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl (revision 29124) +++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/getSmfForThisBatch.pl (revision 29124) @@ -0,0 +1,82 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use ippToPsps::Gpc1Db; +use ippToPsps::IppToPspsDb; +use PS::IPP::Config 1.01 qw( :standard ); + +my $camera = undef; +my $singleBatch = undef; +my $verbose = undef; +my $save_temps = undef; + +GetOptions( + 'batch|b=s' => \$singleBatch, + 'verbose|v' => \$verbose, + 'save_temps|t' => \$save_temps, + 'camera|c' => \$camera, + ); + +my $quit = 0; +print "\n*******************************************************************************\n"; +print "* \n"; +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} +if (!defined $singleBatch) { + $quit=1; + print "* REQUIRED: need to provide a batch -b \n"; +} +if (!defined $verbose) { + $camera = "GPC1"; + print "* OPTIONAL: select a camera -c (default = $camera)\n"; +} +if (!defined $verbose) { + $verbose = 0; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; +} +if (!defined $save_temps) { + $save_temps = 0; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; +} + +print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + +my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); +my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); +my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); + + +my $batches; +my $numOfBatches; + +if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);} +if ($numOfBatches < 1) {return 0;} + + +# loop round batches +my $batch; +my $numChecked = 0; +foreach $batch ( @{$batches} ) { + my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) = @{$batch}; + + print "* found $timestamp, $expId, $batchId, $surveyType, $deleted $dvoDb\n"; + + my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId); + if (!$nebPath) { print "* Could not determind neb path for SMF\n"; exit; } + + # get real filename from neb 'key' + my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0; + my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; + + print "$realFile\n"; + +} + + Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/BatchManager.pm =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/BatchManager.pm (revision 29124) +++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/BatchManager.pm (revision 29124) @@ -0,0 +1,95 @@ +#!/usr/bin/perl -w + +package ippToPsps::BatchManager; + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + + my $class = shift; + my $self = { + _ippToPspsDb => shift, + _path => shift, + _verbose => shift, + _save_temps => shift, + }; + + bless $self, $class; + return $self; +} + +######################################################################################## +# +# Returns batch name from batch_id +# +######################################################################################## +sub getBatchName { + my ($self, $batchId) = @_; + + return sprintf("B%08d", $batchId); +} + +######################################################################################## +# +# Returns name of tarred and zipped batch file +# +######################################################################################## +sub getBatchFileName { + my ($self, $batchId) = @_; + + my $batchName = $self->getBatchName($batchId); + return "$batchName.tar.gz"; +} + +######################################################################################## +# +# Returns full path of batch +# +######################################################################################## +sub getFullBatchPath { + my ($self, $batchId) = @_; + + my $batches; + my $numOfBatches = $self->{_ippToPspsDb}->getBatch($batchId, \$batches); + if ($numOfBatches < 1) {return "unknown";} + + my $batch; + my ($timestamp, $expId, $batchIdi, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged); + foreach $batch ( @{$batches} ) { + ($timestamp, $expId, $batchIdi, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) = @{$batch}; + + + } + + my $batchFile = $self->getBatchFileName($batchId); + return "$self->{_path}/$dvoDb/$batchFile"; +} + +######################################################################################## +# +# Deletes a batch +# +######################################################################################## +sub deleteBatch { + my ($self, $batchId) = @_; + + my $command = "rm " . $self->getFullBatchPath($batchId); + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $command, verbose => $self->{_verbose}); + if (!$success) { + print "* Unable to run: $command\n"; + return 0; + } + + return 1; +} + +1; Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 29124) @@ -15,13 +15,11 @@ # ########################################################################### -sub getBatchList { - my ($self, $exposures, $fromTime, $toTime) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<= '$fromTime' + WHERE created >= '$fromTime' AND created <= '$toTime'; SQL @@ -29,6 +27,6 @@ # TODO remove date restriction $query->execute; - ${$exposures} = $query->fetchall_arrayref(); - my $count = scalar @{${$exposures}}; + ${$batches} = $query->fetchall_arrayref(); + my $count = scalar @{${$batches}}; printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es"); @@ -41,18 +39,16 @@ # ########################################################################### -sub getSingleBatch{ - my ($self, $batch_id, $exposures) = @_; - - my $query = $self->{_db}->prepare(<{_db}->prepare(<execute; - ${$exposures} = $query->fetchall_arrayref(); - my $count = scalar @{${$exposures}}; + ${$batches} = $query->fetchall_arrayref(); + my $count = scalar @{${$batches}}; printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es"); @@ -149,6 +145,4 @@ my $processed = $query->fetchrow_array(); - - if ($processed) {print "* Exposure ID '$expId' has already been processed\n";} return $processed; @@ -424,3 +418,18 @@ return $row[0]; } + +sub insetDetection { + my ($self, $expId, $imageID, $objID,$ippObjID, $ippDetectID) = @_; + + my $query = $self->{_db}->prepare(<execute; + +} + 1 Index: /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps_run.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippToPsps/perl/ippToPsps_run.pl (revision 29124) @@ -17,5 +17,5 @@ # globals -my $camera = 'GPC1'; +my $camera = undef; my $batchType = undef; my $dvoLocation = undef; @@ -33,4 +33,5 @@ # get user args GetOptions( + 'camera|c' => \$camera, 'output|o=s' => \$output, 'batch|b=s' => \$batchType, @@ -63,4 +64,8 @@ print "* REQUIRED: need to provide a DVO Db -d \n"; } +if (!defined $camera) { + $camera = "GPC1"; + print "* OPTIONAL: select a camera -c (default = $camera)\n"; +} if (!defined $singleExpId) { @@ -175,6 +180,10 @@ if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) { - if ($force) {print "* Forcing....\n";} - else {next}; + if ($force) {print "* Already processed '$expId', but forcing....\n";} + else { + + print "* Exposure ID '$expId' has already been processed\n"; + next + }; } Index: /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippToPsps/src/ippToPspsBatchDetection.c (revision 29124) @@ -305,9 +305,10 @@ numInvalidFlux++; } - - // store max/min objID - if (objID[s] > maxObjID) maxObjID = objID[s]; - if (objID[s] < minObjID) minObjID = objID[s]; - + else { + + // store max/min objID + if (objID[s] > maxObjID) maxObjID = objID[s]; + if (objID[s] < minObjID) minObjID = objID[s]; + } totalDetections++; } Index: /branches/eam_branches/ipp-20100823/ippToPsps/xmlschema/BatchManifest.xsd =================================================================== --- /branches/eam_branches/ipp-20100823/ippToPsps/xmlschema/BatchManifest.xsd (revision 29124) +++ /branches/eam_branches/ipp-20100823/ippToPsps/xmlschema/BatchManifest.xsd (revision 29124) @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: /branches/eam_branches/ipp-20100823/ippTools/configure.ac =================================================================== --- /branches/eam_branches/ipp-20100823/ippTools/configure.ac (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTools/configure.ac (revision 29124) @@ -19,5 +19,5 @@ PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) -PKG_CHECK_MODULES([PPSTAMP], [ippdb >= 1.1.63]) +PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${PPSTAMP_CFLAGS=} ${IPPDB_CFLAGS=}" Index: /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_tocleanup.sql =================================================================== --- /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_tocleanup.sql (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTools/share/magicdstool_tocleanup.sql (revision 29124) @@ -1,4 +1,5 @@ SELECT magic_ds_id, + stage, magicDSRun.state, magicDSRun.outroot, @@ -9,4 +10,5 @@ WHERE magicDSRun.state = 'goto_cleaned' -- XXX: need to add fault to magicDSRun +-- XXX: the database has been updated, but fault isn't yet used -- AND magicDSRun.fault = 0 Index: /branches/eam_branches/ipp-20100823/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20100823/ippTools/src/disttool.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTools/src/disttool.c (revision 29124) @@ -519,5 +519,5 @@ PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; PXOPT_COPY_STR(config->args, where, "-state", "distRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-dist_group", "distTarget.dist_group", "=="); Index: /branches/eam_branches/ipp-20100823/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20100823/ippTools/src/disttoolConfig.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTools/src/disttoolConfig.c (revision 29124) @@ -76,6 +76,6 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "value for stage", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "value for state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "limit updates to data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "limit updates to data_group (LIKE comparison)", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "limit updates to data_group", NULL); psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_begin", 0, "limit updates by time_stamp (>=)", NULL); Index: /branches/eam_branches/ipp-20100823/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20100823/ippTools/src/staticskytool.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippTools/src/staticskytool.c (revision 29124) @@ -290,5 +290,5 @@ workdir, label, - data_group, + data_group ? data_group : label, dist_group, reduction, Index: /branches/eam_branches/ipp-20100823/ippconfig/recipes/fitstypes.mdc =================================================================== --- /branches/eam_branches/ipp-20100823/ippconfig/recipes/fitstypes.mdc (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippconfig/recipes/fitstypes.mdc (revision 29124) @@ -107,4 +107,6 @@ BSCALE F32 0.1 # Disk is stored as 1/10th sec BZERO F32 3276.8 # Store unsigned zero as zero + # switch to NONE to work around ticket 1411 + #COMPRESSION STR NONE COMPRESSION STR GZIP TILE.X S32 0 Index: /branches/eam_branches/ipp-20100823/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20100823/ippconfig/recipes/nightly_science.config (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippconfig/recipes/nightly_science.config (revision 29124) @@ -39,5 +39,4 @@ UNRECOVERABLE_QUALITY MULTI -UNRECOVERABLE_QUALITY S16 0 UNRECOVERABLE_QUALITY S16 4007 Index: /branches/eam_branches/ipp-20100823/ippconfig/recipes/ppSub.config =================================================================== --- /branches/eam_branches/ipp-20100823/ippconfig/recipes/ppSub.config (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippconfig/recipes/ppSub.config (revision 29124) @@ -59,5 +59,5 @@ INNER S32 5 # Inner half-size for SPAM and FRIES kernels RINGS.ORDER S32 2 # Polynomial order for RINGS kernels -PENALTY F32 1.0 # Penalty for wideness +PENALTY F32 16.0 # Penalty for wideness BIN1 S32 4 # Binning factor for first level BIN2 S32 4 # Binning factor for second level Index: /branches/eam_branches/ipp-20100823/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/ipp-20100823/ippconfig/recipes/psphot.config (revision 29123) +++ /branches/eam_branches/ipp-20100823/ippconfig/recipes/psphot.config (revision 29124) @@ -188,5 +188,7 @@ #PGAUSS_PSF EXTENDED_SOURCE_MODEL PS_MODEL_PGAUSS 20.0 TRUE #QGAUSS_PSF EXTENDED_SOURCE_MODEL PS_MODEL_QGAUSS 20.0 TRUE - SERSIC_PSF EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 20.0 TRUE + EXP_PCM EXTENDED_SOURCE_MODEL PS_MODEL_EXP 20.0 TRUE + DEV_PCM EXTENDED_SOURCE_MODEL PS_MODEL_DEV 20.0 TRUE + SER_PCM EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 20.0 TRUE END @@ -325,10 +327,9 @@ # Extended source fit parameters STACKPHOT METADATA + EXTENDED_SOURCE_FITS BOOL TRUE # perform any of the aperture-like measurements? EXTENDED_SOURCE_ANALYSIS BOOL TRUE # perform any of the aperture-like measurements? EXTENDED_SOURCE_SN_LIM F32 20.0 EXTENDED_SOURCE_PETROSIAN BOOL TRUE - EXTENDED_SOURCE_ISOPHOTAL BOOL FALSE EXTENDED_SOURCE_ANNULI BOOL TRUE - EXTENDED_SOURCE_KRON BOOL FALSE PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? @@ -337,4 +338,8 @@ RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation OUTPUT.FORMAT STR PS1_SV1 + + EXT_FIT_MAX_RADIUS F32 50.0 + PSF_MODEL STR PS_MODEL_GAUSS + EXT_MODEL STR PS_MODEL_SERSIC END Index: /branches/eam_branches/ipp-20100823/psLib/src/math/psUnaryOp.c =================================================================== --- /branches/eam_branches/ipp-20100823/psLib/src/math/psUnaryOp.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psLib/src/math/psUnaryOp.c (revision 29124) @@ -63,9 +63,9 @@ // Conversion for degrees to radians -#define D2R 0.01745329252111111 /* PI/180 */ - +// #define D2R 0.01745329252111111 /* PI/180 */ +#define D2R 0.01745329251994329 /* PI/180 Corrected. Truncated digits: 576924 */ // Conversion for radians to degrees -#define R2D 57.29577950924861 /* 180.0/PI */ - +// #define R2D 57.29577950924861 /* 180.0/PI */ +#define R2D 57.29577951308232 /* 180.0/PI Correcte. Truncated digits: 087679 */ // Unary SCALAR operations Index: /branches/eam_branches/ipp-20100823/psModules/src/imcombine/pmSubtractionStamps.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/imcombine/pmSubtractionStamps.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/imcombine/pmSubtractionStamps.c (revision 29124) @@ -830,5 +830,5 @@ } -#if 1 +#if 0 { psFits *fits = NULL; Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/models/pmModel_SERSIC.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/models/pmModel_SERSIC.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/models/pmModel_SERSIC.c (revision 29124) @@ -252,5 +252,5 @@ float Rminor = Rmajor * (axes.minor / axes.major); - fprintf (stderr, "guess index: %f : %f, %f -> %f, %f\n", index, axes.major, axes.minor, Rmajor, Rminor); + // fprintf (stderr, "guess index: %f : %f, %f -> %f, %f\n", index, axes.major, axes.minor, Rmajor, Rminor); axes.major = Rmajor; @@ -346,5 +346,5 @@ psF64 radius = axes.major * sqrt (2.0) * pow(zn, 0.5 / PAR[PM_PAR_7]); - fprintf (stderr, "sersic model %f %f, n %f, radius: %f, zn: %f, f/Io: %f, major: %f\n", PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], PAR[PM_PAR_7], radius, zn, flux/PAR[PM_PAR_I0], axes.major); + // fprintf (stderr, "sersic model %f %f, n %f, radius: %f, zn: %f, f/Io: %f, major: %f\n", PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], PAR[PM_PAR_7], radius, zn, flux/PAR[PM_PAR_I0], axes.major); psAssert (isfinite(radius), "fix this code: z should not be nan for %f", PAR[PM_PAR_7]); Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_DV2.c (revision 29124) @@ -186,4 +186,6 @@ psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude", source->psfFluxErr); psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", source->apMag); + // XXX replace with AP_FLUX (& ERR)? psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", source->apMag); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", apRadius); psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", peakMag); @@ -214,4 +216,20 @@ psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY", PS_DATA_F32, "second moments (X*Y)", mxy); psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY", PS_DATA_F32, "second moments (Y*Y)", myy); + + float Mrf = source->moments ? source->moments->Mrf : NAN; + float Mrh = source->moments ? source->moments->Mrh : NAN; + float Krf = source->moments ? source->moments->KronFlux : NAN; + float dKrf = source->moments ? source->moments->KronFluxErr : NAN; + + float Kinner = source->moments ? source->moments->KronFinner : NAN; + float Kouter = source->moments ? source->moments->KronFouter : NAN; + + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1", PS_DATA_F32, "first radial moment", Mrf); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH", PS_DATA_F32, "half radial moment", Mrh); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX", PS_DATA_F32, "Kron Flux (in 2.5 R1)", Krf); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR", PS_DATA_F32, "Kron Flux Error", dKrf); + + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER", PS_DATA_F32, "Kron Flux (in 1.0 R1)", Kinner); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER", PS_DATA_F32, "Kron Flux (in 4.0 R1)", Kouter); psMetadataAdd (row, PS_LIST_TAIL, "DIFF_NPOS", PS_DATA_S32, "nPos (n pix > 3 sigma)", diffStats.nGood); Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_SV1.c (revision 29124) @@ -198,4 +198,5 @@ psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental flux", source->psfFluxErr); psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", source->apMag); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW", PS_DATA_F32, "magnitude in reported aperture", source->apMagRaw); psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", apRadius); psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", peakMag); Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceIO_CMF_PS1_V3.c (revision 29124) @@ -179,4 +179,5 @@ psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW", PS_DATA_F32, "magnitude in reported aperture", source->apMagRaw); psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", apRadius); + // XXX need ap_mag error psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", peakMag); psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG", PS_DATA_F32, "PSF Magnitude using supplied calibration", calMag); @@ -204,4 +205,18 @@ float Myy = source->moments ? source->moments->Myy : NAN; + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XX", PS_DATA_F32, "second moments (X^2)", Mxx); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY", PS_DATA_F32, "second moments (X*Y)", Mxy); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY", PS_DATA_F32, "second moments (Y*Y)", Myy); + + float M_c3 = source->moments ? 1.0*source->moments->Mxxx - 3.0*source->moments->Mxyy : NAN; + float M_s3 = source->moments ? 3.0*source->moments->Mxxy - 1.0*source->moments->Myyy : NAN; + float M_c4 = source->moments ? 1.0*source->moments->Mxxxx - 6.0*source->moments->Mxxyy + 1.0*source->moments->Myyyy : NAN; + float M_s4 = source->moments ? 4.0*source->moments->Mxxxy - 4.0*source->moments->Mxyyy : NAN; + + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3C", PS_DATA_F32, "third momemt cos theta", M_c3); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3S", PS_DATA_F32, "third momemt sin theta", M_s3); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4C", PS_DATA_F32, "fourth momemt cos theta", M_c4); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4S", PS_DATA_F32, "fourth momemt sin theta", M_s4); + float Mrf = source->moments ? source->moments->Mrf : NAN; float Mrh = source->moments ? source->moments->Mrh : NAN; @@ -211,18 +226,4 @@ float Kinner = source->moments ? source->moments->KronFinner : NAN; float Kouter = source->moments ? source->moments->KronFouter : NAN; - - float M_c3 = source->moments ? 1.0*source->moments->Mxxx - 3.0*source->moments->Mxyy : NAN; - float M_s3 = source->moments ? 3.0*source->moments->Mxxy - 1.0*source->moments->Myyy : NAN; - float M_c4 = source->moments ? 1.0*source->moments->Mxxxx - 6.0*source->moments->Mxxyy + 1.0*source->moments->Myyyy : NAN; - float M_s4 = source->moments ? 4.0*source->moments->Mxxxy - 4.0*source->moments->Mxyyy : NAN; - - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XX", PS_DATA_F32, "second moments (X^2)", Mxx); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY", PS_DATA_F32, "second moments (X*Y)", Mxy); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY", PS_DATA_F32, "second moments (Y*Y)", Myy); - - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3C", PS_DATA_F32, "third momemt cos theta", M_c3); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M3S", PS_DATA_F32, "third momemt sin theta", M_s3); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4C", PS_DATA_F32, "fourth momemt cos theta", M_c4); - psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_M4S", PS_DATA_F32, "fourth momemt sin theta", M_s4); psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1", PS_DATA_F32, "first radial moment", Mrf); Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceMoments.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceMoments.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourceMoments.c (revision 29124) @@ -77,4 +77,7 @@ // well: psF32 sky = 0.0; + if (source->moments == NULL) { + source->moments = pmMomentsAlloc(); + } // XXX if (source->moments == NULL) { // XXX source->moments = pmMomentsAlloc(); Index: /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourcePhotometry.c =================================================================== --- /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourcePhotometry.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psModules/src/objects/pmSourcePhotometry.c (revision 29124) @@ -148,4 +148,5 @@ // for PSFs, correct both apMag and psfMag to same system, consistent with infinite flux star in aperture RADIUS + // XXX add a flag for "ap_mag is corrected?" if ((mode & PM_SOURCE_PHOT_APCORR) && isPSF && psf && psf->ApTrend) { // the source peak pixel is guaranteed to be on the image, and only minimally different from the source center @@ -292,4 +293,7 @@ // measure apMag + // XXX save the apFlux and apFluxErr + // XXX note that these fluxes/mags are uncorrected for masked pixels + // XXX raise a bit if the aperture has a masked pixel (not marked)? for (int iy = 0; iy < image->numRows; iy++) { for (int ix = 0; ix < image->numCols; ix++) { Index: /branches/eam_branches/ipp-20100823/psconfig/tagsets/ipp-2.9.perl =================================================================== --- /branches/eam_branches/ipp-20100823/psconfig/tagsets/ipp-2.9.perl (revision 29123) +++ /branches/eam_branches/ipp-20100823/psconfig/tagsets/ipp-2.9.perl (revision 29124) @@ -92,8 +92,10 @@ 80 Abstract::Meta::Class Abstract-Meta-Class-0.13.tar.gz 0 NONE NONE 81 DBIx::Connection DBIx-Connection-0.13.tar.gz 0 NONE NONE - 82a Test::Pod Test-Pod-1.40.tar.gz 0 NONE NONE - 82c XML::NamespaceSupport XML-NamespaceSupport-1.10.tar.gz 0 --skip NONE - 82b XML::SAX XML-SAX-0.96.tar.gz 0 NONE Y - 82d Simple::SAX::Serializer Simple-SAX-Serializer-0.05.tar.gz 0 NONE NONE + 82a Pod::Escapes Pod-Escapes-1.04.tar.gz 0 NONE NONE + 82b Pod::Simple Pod-Simple-3.14.tar.gz 0 NONE NONE + 82c Test::Pod Test-Pod-1.40.tar.gz 0 NONE NONE + 82d XML::NamespaceSupport XML-NamespaceSupport-1.10.tar.gz 0 --skip NONE + 82e XML::SAX XML-SAX-0.96.tar.gz 0 NONE Y + 82f Simple::SAX::Serializer Simple-SAX-Serializer-0.05.tar.gz 0 NONE NONE 83 Test::Distribution Test-Distribution-2.00.tar.gz 0 NONE NONE 84 Test::DBUnit Test-DBUnit-0.20.tar.gz 0.20 NONE NONE Index: /branches/eam_branches/ipp-20100823/psphot/doc/notes.20100715.txt =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/doc/notes.20100715.txt (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/doc/notes.20100715.txt (revision 29124) @@ -1,2 +1,18 @@ + +2010.08.24 + + Remaining work to be done: + + * test and turn on CR masking + * double-check source size analysis: + * are we doing the right thing with sources flagged as bad in 'rough'? + * are we doing the right thing for SAT, CR, EXT, PSF after SourceSize? + * convert CR_SIGMA and EXT_SIGMA to probabilities + * example results of model fits vs fake inputs + * what is the right solution for the stack PSF photometry? + * Nigel sees 4-6% + * we see 1.9% at the worst + * non-poisson errors are clearly better -- check on the chisq values + * why is QGAUSS so good for the one example (better for poisson, if trendy) 2010.08.12 Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotBlendFit.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotBlendFit.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotBlendFit.c (revision 29124) @@ -66,8 +66,15 @@ float fitMinTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MIN_TOL"); // Fit tolerance - assert (status && isfinite(fitMinTol) && fitMinTol > 0); + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + psAbort("PSF_FIT_MIN_TOL (and PSF_FIT_TOL) not defined or positive"); + } + } float fitMaxTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_TOL"); // Fit tolerance - assert (status && isfinite(fitMaxTol) && fitMaxTol > 0); + if (!status || !isfinite(fitMaxTol) || fitMaxTol <= 0) { + fitMaxTol = 1.0; + } bool poisson = psMetadataLookupBool(&status, recipe, "POISSON.ERRORS.PHOT.LMM"); // Poisson errors? Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceAnalysis.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceAnalysis.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceAnalysis.c (revision 29124) @@ -121,4 +121,6 @@ if (source->peak->y > AnalysisRegion.y1) continue; + // fprintf (stderr, "xsrc: %f, %f : %f\n", source->peak->xf, source->peak->yf, source->peak->SN); + // replace object in image if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { @@ -178,4 +180,6 @@ } + // fprintf (stderr, "xsrc : tried %ld objects\n", sources->n); + return true; } Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceFits.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceFits.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotExtendedSourceFits.c (revision 29124) @@ -200,5 +200,5 @@ } psFree(job); - } + } } psFree (cellGroups); @@ -270,4 +270,5 @@ // set the radius based on the footprint (also sets the mask pixels) if (!psphotSetRadiusFootprint(&radius, readout, source, markVal, 1.0)) { + fprintf (stderr, "skipping (1) %f, %f\n", source->peak->xf, source->peak->yf); psFree (fitOptions) return false; @@ -280,4 +281,5 @@ // XXX save the psf-based moments for output if (!pmSourceMoments (source, radius, 0.0, 0.0, maskVal)) { + fprintf (stderr, "skipping (2) %f, %f\n", source->peak->xf, source->peak->yf); // subtract the best fit from the object, leave local sky pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); @@ -289,8 +291,13 @@ psImage *modelFluxStart = psMemIncrRefCounter (source->modelFlux); if (!modelFluxStart) { - // XXX raise an error of some kind? - continue; + pmSourceCacheModel (source, maskVal); + modelFluxStart = psMemIncrRefCounter (source->modelFlux); + if (!modelFluxStart) { + fprintf (stderr, "skipping (3) %f, %f\n", source->peak->xf, source->peak->yf); + // XXX raise an error of some kind? + continue; + } } - + if (savePics) { psphotSaveImage (NULL, readout->image, "image.xp.fits"); @@ -317,4 +324,6 @@ float SNlim = psMetadataLookupF32 (&status, model, "SNLIM_VALUE"); assert (status); + + // fprintf (stderr, "xfit: %f, %f : %f\n", source->peak->xf, source->peak->yf, source->peak->SN); // limit selection to some SN limit @@ -449,4 +458,6 @@ psFree (fitOptions); + // fprintf (stderr, "xfit : tried %ld objects\n", sources->n); + // change the value of a scalar on the array (wrap this and put it in psArray.h) scalar = job->args->data[7]; Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceFits.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceFits.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceFits.c (revision 29124) @@ -506,5 +506,5 @@ // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5); pmSourceFitModel (source, model, &options, maskVal); - fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0); @@ -570,5 +570,5 @@ // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5); pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); - fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0); @@ -613,5 +613,5 @@ pmSourceFitModel (source, model, &options, maskVal); - fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); chiSquare[i] = model->chisqNorm; @@ -681,5 +681,5 @@ pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); # endif - fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); chiSquare[i] = model->chisq; Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceSize.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceSize.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotSourceSize.c (revision 29124) @@ -366,7 +366,7 @@ float nSigmaMYY = (Myy - psfClump->Y) / hypot(psfClump->dY, psfClump->Y*psfClump->Y*source->errMag); - fprintf (stderr, "%f %f : Mxx: %f, Myy: %f, dx: %f, dy: %f, psfMag: %f, apMag: %f, dMag: %f, errMag: %f, nSigmaMag: %f, nSigmaMxx: %f, nSigmaMyy: %f\n", - source->peak->xf, source->peak->yf, Mxx, Myy, source->peak->xf - source->moments->Mx, source->peak->yf - source->moments->My, - source->psfMag, apMag, dMag, source->errMag, nSigmaMAG, nSigmaMXX, nSigmaMYY); + // fprintf (stderr, "%f %f : Mxx: %f, Myy: %f, dx: %f, dy: %f, psfMag: %f, apMag: %f, dMag: %f, errMag: %f, nSigmaMag: %f, nSigmaMxx: %f, nSigmaMyy: %f\n", + // source->peak->xf, source->peak->yf, Mxx, Myy, source->peak->xf - source->moments->Mx, source->peak->yf - source->moments->My, + // source->psfMag, apMag, dMag, source->errMag, nSigmaMAG, nSigmaMXX, nSigmaMYY); // XXX double check on ths stuff!! partially-masked sources are more likely to be mis-measured PSFs Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFs.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFs.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFs.c (revision 29124) @@ -106,5 +106,5 @@ rescaleData(readoutOut, config, options, index); - dumpImage(readoutOut, readoutSrc, index, "convolved"); + // dumpImage(readoutOut, readoutSrc, index, "convolved"); return true; Index: /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFsUtils.c =================================================================== --- /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFsUtils.c (revision 29123) +++ /branches/eam_branches/ipp-20100823/psphot/src/psphotStackMatchPSFsUtils.c (revision 29124) @@ -346,6 +346,6 @@ if (!fake) goto escape; - dumpImage(fake, readoutSrc, index, "fake"); - dumpImage(readoutSrc, readoutSrc, index, "real"); + // dumpImage(fake, readoutSrc, index, "fake"); + // dumpImage(readoutSrc, readoutSrc, index, "real"); if (threads) pmSubtractionThreadsInit(); @@ -380,6 +380,6 @@ } - dumpImage(readoutOut, readoutSrc, index, "conv"); - dumpImageDiff(readoutOut, fake, readoutSrc, index, "diff"); + // dumpImage(readoutOut, readoutSrc, index, "conv"); + // dumpImageDiff(readoutOut, fake, readoutSrc, index, "diff"); psFree(fake); Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am (revision 29123) +++ /branches/eam_branches/ipp-20100823/pstamp/scripts/Makefile.am (revision 29124) @@ -19,4 +19,5 @@ pstamp_webrequest.pl \ pstamp_get_image_job.pl \ + psmkreq \ pstamp_checkdependent.pl \ request_finish.pl \ Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/pstamp/scripts/detectability_respond.pl (revision 29124) @@ -98,4 +98,8 @@ my $key = shift(@key_values); my $val = shift(@key_values); + # if we have wisdom, then we should have updated already. If not, we'll bomb out later in the code. + if ($key eq 'FAULT') { + $val = 0; + } $query{$fpa_id}{$key}[$i] = $val; } @@ -148,36 +152,36 @@ # Determine the query style for this fpa_id if ($fpa_id =~ /o.*g.*o/) { - $query_style = 'byexp'; - } + $query_style = 'byexp'; + } elsif ($fpa_id =~ /\d+/) { - $query_style = 'byid'; - } + $query_style = 'byid'; + } else { - exit_with_failure(21,"Parse error in request file"); - } + exit_with_failure(21,"Parse error in request file"); + } # Confirm that we only have one stage/filter/mjd for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) { - $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; - $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; - $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; - } + $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1; + $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1; + $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1; + } if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) { $stage = (keys(%{ $temp_hash{STAGE} }))[0]; } else { - exit_with_failure(21,"Too many STAGEs specified"); - } + exit_with_failure(21,"Too many STAGEs specified"); + } if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) { - $filter = (keys(%{ $temp_hash{FILTER} }))[0]; - } + $filter = (keys(%{ $temp_hash{FILTER} }))[0]; + } else { - exit_with_failure(21,"Too many FILTERs specified"); - } + exit_with_failure(21,"Too many FILTERs specified"); + } if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) { - $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; - } + $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0]; + } else { - exit_with_failure(21,"Too many MJD-OBS specified"); - } + exit_with_failure(21,"Too many MJD-OBS specified"); + } # Set common request components my $option_mask |= 1; @@ -200,4 +204,24 @@ $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i]; $rowList[$i]->{COORD_MASK} = 0; + # Set default values + $query{$fpa_id}{BAD_COMPONENT}[$i] = 1; + $query{$fpa_id}{IMAGE}[$i] = 'no_image'; + $query{$fpa_id}{MASK}[$i] = 'no_mask'; + $query{$fpa_id}{WEIGHT}[$i] = 'no_weight'; + $query{$fpa_id}{PSF}[$i] = 'no_psf'; + + $query{$fpa_id}{STAGE_ID}[$i] = 'no_id'; + $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb'; + $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic'; + $query{$fpa_id}{MAGICKED}[$i] = 'no_magic'; + $query{$fpa_id}{CATALOG}[$i] = 'no_catalog'; + $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component'; + $query{$fpa_id}{CLASS_ID}[$i] = 'no_class'; + + $query{$fpa_id}{STATE}[$i] = 'no_state'; + $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate'; + $query{$fpa_id}{FAULT}[$i] = 'no_fault'; + $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate'; + } @@ -217,4 +241,5 @@ $value = join ' ', @{ $this_image_ref->{$key} }; } + # print "$this_image_ref $key $value\n"; foreach my $valid_index (@{ $this_image_ref->{row_index} }) { @@ -226,4 +251,5 @@ $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb}; $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic; + $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0; if (exists($this_image_ref->{astrom})) { @@ -231,6 +257,6 @@ } else { - $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; - } + $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf}; + } if (exists($this_image_ref->{class_id})) { $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id}; @@ -301,8 +327,10 @@ } print WISDOM "\n"; - @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = - ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], - $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); - push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; + if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) { + @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = + ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i], + $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]); + push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i; + } } } @@ -356,5 +384,12 @@ my $stage = $query{$fpa_id}{STAGE}[$index]; # if there's a fault, then we can't process this image. - if ($fault != 0) { + if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) { + $query{$fpa_id}{PROC_ERROR}[$index] = 23; + + $query{$fpa_id}{NPIX}[$index] = 0; + $query{$fpa_id}{QFACTOR}[$index] = 0.0; + $query{$fpa_id}{FLUX}[$index] = 0.0; + $query{$fpa_id}{FLUX_SIG}[$index] = 0.0; + next; } @@ -384,6 +419,6 @@ my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line; print $targetfile "$r_x $r_y\n"; - if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&& - ($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) { + if ((abs($r_ra - $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)&& + (abs($r_dec - $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)) { $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x; $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y; @@ -392,5 +427,5 @@ else { $error_code = $PS_EXIT_PROG_ERROR; - my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code", + my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) $error_code", $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index], $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index], Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq (revision 29124) +++ /branches/eam_branches/ipp-20100823/pstamp/scripts/psmkreq (revision 29124) @@ -0,0 +1,374 @@ +#!/bin/env perl +### +### pstamprequest +### +### Program to make a postage stamp request table for a set of coordinates +### + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions ) ; # :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); +use File::Basename qw(basename); +use IPC::Cmd 0.36 qw( can_run run ); +use Carp; +use POSIX; + +my $verbose; +my $save_temps; + +# list file columns +# RA DEC FILTER MJD_MIN MJD_MAX + +my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); + +my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse); +my ($unconvolved, $use_imfile_id, $no_wait); + +my $default_size = 100; +my $job_type = 'stamp'; +my $req_type = 'bycoord'; +my $stage = 'chip'; +my $option_mask; +my $width = $default_size; +my $height = $default_size; +my $project = 'gpc1'; +my $coord_mask; +my $pixcenter; +my $arcseconds; + +my $id; +my $tess_id = 'null'; +my $component = 'null'; +my $data_group = 'null'; +my $filter; +my $mjd_min = 0; +my $mjd_max = 0; +my $comment; + +my $missing_tools; +my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +GetOptions( + 'list=s' => \$list, # list of coordinates if undef ra and dec or x and y are required + 'ra=s' => \$ra, # + 'dec=s' => \$dec, + 'x=s' => \$x, + 'y=s' => \$y, + 'width=i' => \$width, + 'height=i' => \$height, + 'pixcenter' => \$pixcenter, + 'arcseconds' => \$arcseconds, + 'coord_mask=i' => \$coord_mask, + 'job_type=s' => \$job_type, + 'output=s' => \$output, + 'req_name=s' => \$req_name, + 'req_name_base=s' => \$req_name_base, + 'stage=s' => \$stage, + 'project=s' => \$project, + 'req_type=s' => \$req_type, + 'id=s' => \$id, + 'tess_id=s' => \$tess_id, + 'component=s' => \$component, + 'data_group=s' => \$data_group, + 'filter=s' => \$filter, + 'mjd_min=s' => \$mjd_min, + 'mjd_max=s' => \$mjd_max, + 'comment=s' => \$comment, + + 'option_mask=i' => \$option_mask, + 'image' => \$image, + 'mask' => \$mask, + 'variance' => \$variance, + 'cmf' => \$cmf, + 'psf' => \$psf, + 'backmdl' => \$backmdl, + 'inverse' => \$inverse, + 'unconvolved' => \$unconvolved, + 'use_imfile_id' => \$use_imfile_id, + 'do_not_wait' => \$no_wait, + + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, +) or pod2usage(2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( -msg => "Required options: --req_name | --req_name_base --stage ") + unless (defined $req_name or defined $req_name_base) + and defined $stage; + +pod2usage( -msg => "Invalid job_type: $job_type", -exitval =>1 ) + unless ($job_type eq 'stamp' or $job_type eq 'get_image'); + +if ($job_type eq 'stamp') { + if (defined $list) { + pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 ) + if defined $x or defined $y; + } elsif (!$pixcenter) { + pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 ) + unless (defined $ra and defined $dec); + # to simplify code we just use $x and $y from here + $x = $ra; + $y = $dec; + } else { + pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 ) + unless (defined $x and defined $y); + } +} else { + pod2usage( -msg => "req_type must be byid or byexp for get_image requests", + -exitval =>1 ) unless $req_type eq 'byid' or $req_type eq 'byexp'; + $x = 0; + $y = 0; +} + +if ($req_type eq 'byid') { + pod2usage( -msg => "ID required for get_image requests", -exitval =>1 ) + if !defined $id; + die("ID must be number for byid requests") if !validID($id); +} elsif ($req_type eq 'byexp') { + pod2usage( -msg => "ID (exp_name) required for byexp requests", -exitval =>1 ) + if !defined $id; +} elsif ($req_type eq 'bydiff') { + pod2usage( -msg => "ID (exp_name) required for bydiff requests", -exitval =>1 ) + if !defined $id; +} elsif ($req_type eq 'byskycell') { + pod2usage( -msg => "tess_id and skycell_id required for byskycell requests", -exitval =>1 ) + if !(defined $tess_id and defined $component); +} elsif ($req_type ne 'bycoord') { + pod2usage( -msg => "$req_type is not a valid req_type", -exitval =>1 ); +} + +$id = 0 if !$id; + +unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') { + die "$stage is not a valid value for stage\n"; +} + +checkFilter($filter, 'null', $filter) if $filter; +checkMJD($mjd_min, 0, "") if $mjd_min; +checkMJD($mjd_max, 0, "" ) if $mjd_min; + +# user supplied option mask takes precedence +if (!defined $option_mask) { + $option_mask = 0; + if ($job_type eq 'stamp') { + $option_mask |= $PSTAMP_SELECT_IMAGE if $image; + $option_mask |= $PSTAMP_SELECT_MASK if $mask; + $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance; + $option_mask = $PSTAMP_SELECT_IMAGE if $option_mask == 0; + + # if no image was requested make a stamp of the image + + $option_mask |= $PSTAMP_SELECT_CMF if $cmf; + $option_mask |= $PSTAMP_SELECT_PSF if $psf; + $option_mask |= $PSTAMP_SELECT_BACKMDL if $backmdl; + $option_mask |= $PSTAMP_SELECT_UNCONV if $unconvolved; + $option_mask |= $PSTAMP_USE_IMFILE_ID if $use_imfile_id; + $option_mask |= $PSTAMP_SELECT_INVERSE if $inverse; + } + $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_wait; +} + + +# user supplied coord_mask takes precedence +if (!defined $coord_mask) { + $coord_mask = 0; + if ($pixcenter) { + $coord_mask |= $PSTAMP_CENTER_IN_PIXELS; + } + if (!$arcseconds) { + $coord_mask |= $PSTAMP_RANGE_IN_PIXELS; + } +} + + +# if req_name is not supplied use the current date and time to build one off of the base +if (!$req_name) { + my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; + $req_name .= $req_name_base . $datestr; +} + +# ok ready to go + +my $rows; +if ($list) { + $rows = readList($list); +} else { + $rows = []; + push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max); +} + +my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps); +print $tdf "$req_name 1\n"; +my $rownum = 0; +foreach my $row (@$rows) { + $rownum++; + my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height" + . " $coord_mask $job_type $option_mask $project $req_type" + . " $stage $id $tess_id $component $data_group" + . " $row->{filter} $row->{mjd_min} $row->{mjd_max}"; + + if ($row->{comment} and $row->{comment} ne '') { + $line .= " |$row->{comment}" ; + } else { + $line .= " |$comment" if $comment; + } + + print "$line\n" if $verbose; + print $tdf "$line\n"; +} +close $tdf; + +{ + my $command = "$pstamp_request_file --input $table_def_name --req_name $req_name"; + $command .= " --output $output" if $output; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + print STDERR @$stderr_buf; + exit $error_code >> 8; + } +} + +exit 0; + +sub readList { + my $file = shift; + open IN, "<$file" or die "failed to open $file for input\n"; + + my @rows; + + my $linenumber = 0; + foreach my $line () { + $linenumber++; + chomp $line; + next if !$line; + next if ($line =~ /^#/); + + my ($spec, $comment) = split /\|/, $line; + $comment = '' if !defined $comment; + + my @vals = split " ", $spec; + push @rows, buildRow("at line number $linenumber", $comment, @vals); + } + close IN; + + return \@rows; +} + +sub buildRow { + my $linenumber = shift; + my $comment = shift; + my @vals = @_; + + my $row = {}; + $row->{ra} = checkRA($vals[0], $linenumber); + $row->{dec} = checkDEC($vals[1], $linenumber); + $row->{filter} = checkFilter($vals[2], $filter, $linenumber); + $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);; + $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);; + $row->{comment} = $comment; + + return $row; +} + +sub validNumber { + my $val = shift; + + return 0 if !defined $val; + + return ($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/); +} + +sub validID +{ + my $val = shift; + + return 0 if !$val; + + return ! ($val =~ /\D/); +} + +sub checkCoord { + my $c = shift; + my $linenumber = shift; + + my $result; + if ($c =~ /\:/) { + # sexagesmial format not valid for pixel coordinates + die "invalid coordinate value $c found $linenumber\n" if $pixcenter; + + my ($d, $m, $s) = split '\:', $c; + die "invalid coordinate value $c found $linenumber\n" + unless validNumber($d) and validNumber($m) and validNumber($s); + my $sign; + if ($c =~ /^-/) { # checking $d < 0 doesn't work for $d = -0 + $sign = -1; + $d = -$d; + } else { + $sign = 1; + } + $result = $sign * ($d + ($m + $s/60.) /60.); + } else { + die "$c is not valid coordinate value $linenumber\n" + unless validNumber($c); + $result = $c; + } + return $result; +} + +sub checkRA { + my $ra = shift; + my $linenumber = shift; + my $checked = checkCoord($ra, $linenumber); + + if ($ra =~ /\:/) { + return $checked * 360. / 24.; + } else { + return $checked; + } +} + +sub checkDEC { + return checkCoord(@_); +} + +sub checkFilter { + my $f = shift; + my $default = shift; + my $linenumber = shift; + + $default = 'null' if !defined $default; + + $f = $default if (!defined$f or $f eq 'null'); + + return $f; +} + +sub checkMJD { + my $mjd = shift; + my $default = shift; + my $linenumber = shift;; + + return $default if (!$mjd) ; + + die "invalid mjd value '$mjd' found $linenumber\n" if !validNumber($mjd); + + return $mjd; +} + +sub checkStage { + my $stage = shift; + +} Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_checkdependent.pl (revision 29124) @@ -644,5 +644,8 @@ # if the input file is already magicked no need to queue destreaking for this chipRun if ($need_magic and !$input_magicked) { - if ($dsRun_state eq 'cleaned') { + if (!defined($dsRun_state) or ($dsRun_state eq 'NULL')) { + print "No magicDSRun for chipRun $stage_id and magic is required\n"; + faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED); + } elsif ($dsRun_state eq 'cleaned') { my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id"; $command .= " -set_label $rlabel" if $rlabel; @@ -658,5 +661,5 @@ } } elsif ($dsRun_state eq 'failed_revert') { - print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update"; + print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n"; faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE); } else { Index: /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamp_webrequest.pl (revision 29124) @@ -4,5 +4,5 @@ # pstampwebrequest.pl: take a postage stamp request command line and process it # -# The arguments are the command line parameters for the program pstamprequest +# The arguments are the command line parameters for the program psmkreq # # Unless the argument -list is provided the output is the request id for the resulting request @@ -70,8 +70,7 @@ my $missing_tools; -my $pstamprequest = can_run('pstamprequest') or (warn "Can't find pstamprequest" and $missing_tools = 1); +my $psmkreq = can_run('psmkreq') or (warn "Can't find psmkreq" and $missing_tools = 1); my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampparse = can_run('pstampparse.pl') or (warn "Can't find pstampparse.pl" and $missing_tools = 1); -my $pstampparser_run = can_run('pstamp_parser_run.pl') or (warn "Can't find pstamp_parser_run.pl" and $missing_tools = 1); if ($missing_tools) { @@ -96,6 +95,5 @@ my $request_file = "$datedir/$request_name.fits"; { - my $command = "$pstamprequest -req_name $request_name -project $project $request_file @ARGV"; - $command .= " -$job_type" if $job_type; # default job_type is pstamp + my $command = "$psmkreq --req_name $request_name --output $request_file @ARGV"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -103,5 +101,5 @@ unless ($success) { print STDERR @$stderr_buf; - die("Unable to perform pstamprequest: $error_code"); + die("Unable to perform psmkreq: $error_code"); } } Index: anches/eam_branches/ipp-20100823/pstamp/scripts/pstamprequest =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/scripts/pstamprequest (revision 29123) +++ (revision ) @@ -1,374 +1,0 @@ -#!/bin/env perl -### -### pstamprequest -### -### Program to make a postage stamp request table for a set of coordinates -### - -use warnings; -use strict; - -use Getopt::Long qw( GetOptions ) ; # :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); -use PS::IPP::Config qw( :standard ); -use PS::IPP::PStamp::RequestFile qw( :standard ); -use PS::IPP::PStamp::Job qw( :standard ); -use File::Temp qw(tempfile); -use File::Basename qw(basename); -use IPC::Cmd 0.36 qw( can_run run ); -use Carp; -use POSIX; - -my $verbose; -my $save_temps; - -# list file columns -# RA DEC FILTER MJD_MIN MJD_MAX - -my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); - -my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse); -my ($unconvolved, $use_imfile_id, $no_wait); - -my $default_size = 100; -my $job_type = 'stamp'; -my $req_type = 'bycoord'; -my $stage = 'chip'; -my $option_mask; -my $width = $default_size; -my $height = $default_size; -my $project = 'gpc1'; -my $coord_mask; -my $pixcenter; -my $arcseconds; - -my $id; -my $tess_id = 'null'; -my $component = 'null'; -my $data_group = 'null'; -my $filter; -my $mjd_min = 0; -my $mjd_max = 0; -my $comment; - -my $missing_tools; -my $pstamp_request_file = can_run('pstamp_request_file') or (warn "Can't find required program pstamp_request_file" and $missing_tools = 1); - -if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); -} - -GetOptions( - 'list=s' => \$list, # list of coordinates if undef ra and dec or x and y are required - 'ra=s' => \$ra, # - 'dec=s' => \$dec, - 'x=s' => \$x, - 'y=s' => \$y, - 'width=i' => \$width, - 'height=i' => \$height, - 'pixcenter' => \$pixcenter, - 'arcseconds' => \$arcseconds, - 'coord_mask=i' => \$coord_mask, - 'job_type=s' => \$job_type, - 'output=s' => \$output, - 'req_name=s' => \$req_name, - 'req_name_base=s' => \$req_name_base, - 'stage=s' => \$stage, - 'project=s' => \$project, - 'req_type=s' => \$req_type, - 'id=s' => \$id, - 'tess_id=s' => \$tess_id, - 'component=s' => \$component, - 'data_group=s' => \$data_group, - 'filter=s' => \$filter, - 'mjd_min=s' => \$mjd_min, - 'mjd_max=s' => \$mjd_max, - 'comment=s' => \$comment, - - 'option_mask=i' => \$option_mask, - 'image' => \$image, - 'mask' => \$mask, - 'variance' => \$variance, - 'cmf' => \$cmf, - 'psf' => \$psf, - 'backmdl' => \$backmdl, - 'inverse' => \$inverse, - 'unconvolved' => \$unconvolved, - 'use_imfile_id' => \$use_imfile_id, - 'do_not_wait' => \$no_wait, - - 'verbose' => \$verbose, - 'save-temps' => \$save_temps, -) or pod2usage(2); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; - -pod2usage( -msg => "Required options: --req_name | --req_name_base --stage ") - unless (defined $req_name or defined $req_name_base) - and defined $stage; - -pod2usage( -msg => "Invalid job_type: $job_type", -exitval =>1 ) - unless ($job_type eq 'stamp' or $job_type eq 'get_image'); - -if ($job_type eq 'stamp') { - if (defined $list) { - pod2usage( -msg => "--ra --dec --x --y are not used with --list", -exitval =>1 ) - if defined $x or defined $y; - } elsif (!$pixcenter) { - pod2usage( -msg => "Required options for stamp requests: --ra and --dec or --list", -exitval =>1 ) - unless (defined $ra and defined $dec); - # to simplify code we just use $x and $y from here - $x = $ra; - $y = $dec; - } else { - pod2usage( -msg => "Required options for stamp requests: --x and --y or --list", -exitval =>1 ) - unless (defined $x and defined $y); - } -} else { - pod2usage( -msg => "req_type must be byid or byexp for get_image requests", - -exitval =>1 ) unless $req_type eq 'byid' or $req_type eq 'byexp'; - $x = 0; - $y = 0; -} - -if ($req_type eq 'byid') { - pod2usage( -msg => "ID required for get_image requests", -exitval =>1 ) - if !defined $id; - die("ID must be number for byid requests") if !validID($id); -} elsif ($req_type eq 'byexp') { - pod2usage( -msg => "ID (exp_name) required for byexp requests", -exitval =>1 ) - if !defined $id; -} elsif ($req_type eq 'bydiff') { - pod2usage( -msg => "ID (exp_name) required for bydiff requests", -exitval =>1 ) - if !defined $id; -} elsif ($req_type eq 'byskycell') { - pod2usage( -msg => "tess_id and skycell_id required for byskycell requests", -exitval =>1 ) - if !(defined $tess_id and defined $component); -} elsif ($req_type ne 'bycoord') { - pod2usage( -msg => "$req_type is not a valid req_type", -exitval =>1 ); -} - -$id = 0 if !$id; - -unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') { - die "$stage is not a valid value for stage\n"; -} - -checkFilter($filter, 'null', $filter) if $filter; -checkMJD($mjd_min, 0, "") if $mjd_min; -checkMJD($mjd_max, 0, "" ) if $mjd_min; - -# user supplied option mask takes precedence -if (!defined $option_mask) { - $option_mask = 0; - if ($job_type eq 'stamp') { - $option_mask |= $PSTAMP_SELECT_IMAGE if $image; - $option_mask |= $PSTAMP_SELECT_MASK if $mask; - $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance; - $option_mask = $PSTAMP_SELECT_IMAGE if $option_mask == 0; - - # if no image was requested make a stamp of the image - - $option_mask |= $PSTAMP_SELECT_CMF if $cmf; - $option_mask |= $PSTAMP_SELECT_PSF if $psf; - $option_mask |= $PSTAMP_SELECT_BACKMDL if $backmdl; - $option_mask |= $PSTAMP_SELECT_UNCONV if $unconvolved; - $option_mask |= $PSTAMP_USE_IMFILE_ID if $use_imfile_id; - $option_mask |= $PSTAMP_SELECT_INVERSE if $inverse; - } - $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_wait; -} - - -# user supplied coord_mask takes precedence -if (!defined $coord_mask) { - $coord_mask = 0; - if ($pixcenter) { - $coord_mask |= $PSTAMP_CENTER_IN_PIXELS; - } - if (!$arcseconds) { - $coord_mask |= $PSTAMP_RANGE_IN_PIXELS; - } -} - - -# if req_name is not supplied use the current date and time to build one off of the base -if (!$req_name) { - my $datestr = strftime "%Y%m%dT%H%M%S", gmtime; - $req_name .= $req_name_base . $datestr; -} - -# ok ready to go - -my $rows; -if ($list) { - $rows = readList($list); -} else { - $rows = []; - push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max); -} - -my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps); -print $tdf "$req_name 1\n"; -my $rownum = 0; -foreach my $row (@$rows) { - $rownum++; - my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height" - . " $coord_mask $job_type $option_mask $project $req_type" - . " $stage $id $tess_id $component $data_group" - . " $row->{filter} $row->{mjd_min} $row->{mjd_max}"; - - if ($row->{comment} and $row->{comment} ne '') { - $line .= " |$row->{comment}" ; - } else { - $line .= " |$comment" if $comment; - } - - print "$line\n" if $verbose; - print $tdf "$line\n"; -} -close $tdf; - -{ - my $command = "$pstamp_request_file --input $table_def_name --req_name $req_name"; - $command .= " --output $output" if $output; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - unless ($success) { - print STDERR @$stderr_buf; - exit $error_code >> 8; - } -} - -exit 0; - -sub readList { - my $file = shift; - open IN, "<$file" or die "failed to open $file for input\n"; - - my @rows; - - my $linenumber = 0; - foreach my $line () { - $linenumber++; - chomp $line; - next if !$line; - next if ($line =~ /^#/); - - my ($spec, $comment) = split /\|/, $line; - $comment = '' if !defined $comment; - - my @vals = split " ", $spec; - push @rows, buildRow("at line number $linenumber", $comment, @vals); - } - close IN; - - return \@rows; -} - -sub buildRow { - my $linenumber = shift; - my $comment = shift; - my @vals = @_; - - my $row = {}; - $row->{ra} = checkRA($vals[0], $linenumber); - $row->{dec} = checkDEC($vals[1], $linenumber); - $row->{filter} = checkFilter($vals[2], $filter, $linenumber); - $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);; - $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);; - $row->{comment} = $comment; - - return $row; -} - -sub validNumber { - my $val = shift; - - return 0 if !defined $val; - - return ($val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/); -} - -sub validID -{ - my $val = shift; - - return 0 if !$val; - - return ! ($val =~ /\D/); -} - -sub checkCoord { - my $c = shift; - my $linenumber = shift; - - my $result; - if ($c =~ /\:/) { - # sexagesmial format not valid for pixel coordinates - die "invalid coordinate value $c found $linenumber\n" if $pixcenter; - - my ($d, $m, $s) = split '\:', $c; - die "invalid coordinate value $c found $linenumber\n" - unless validNumber($d) and validNumber($m) and validNumber($s); - my $sign; - if ($c =~ /^-/) { # checking $d < 0 doesn't work for $d = -0 - $sign = -1; - $d = -$d; - } else { - $sign = 1; - } - $result = $sign * ($d + ($m + $s/60.) /60.); - } else { - die "$c is not valid coordinate value $linenumber\n" - unless validNumber($c); - $result = $c; - } - return $result; -} - -sub checkRA { - my $ra = shift; - my $linenumber = shift; - my $checked = checkCoord($ra, $linenumber); - - if ($ra =~ /\:/) { - return $checked * 360. / 24.; - } else { - return $checked; - } -} - -sub checkDEC { - return checkCoord(@_); -} - -sub checkFilter { - my $f = shift; - my $default = shift; - my $linenumber = shift; - - $default = 'null' if !defined $default; - - $f = $default if (!defined$f or $f eq 'null'); - - return $f; -} - -sub checkMJD { - my $mjd = shift; - my $default = shift; - my $linenumber = shift;; - - return $default if (!$mjd) ; - - die "invalid mjd value '$mjd' found $linenumber\n" if !validNumber($mjd); - - return $mjd; -} - -sub checkStage { - my $stage = shift; - -} Index: /branches/eam_branches/ipp-20100823/pstamp/web/request.php =================================================================== --- /branches/eam_branches/ipp-20100823/pstamp/web/request.php (revision 29123) +++ /branches/eam_branches/ipp-20100823/pstamp/web/request.php (revision 29124) @@ -16,19 +16,6 @@ // XXX This is just a prototype for testing purposes. - -// XXX: change to include pstampconfig.php - -// BEGIN Local configuration - -$WORKDIR = "/data/ipp049.0/pstamp/work"; -$dsroot = "/data/ipp049.0/datastore/dsroot"; -$dbname = "ippRequestServer"; -$dbserver = "ipp049"; - -$PSCONFDIR = "/data/ipp053.0/home/bills/psconfig"; -$PSCONFIG = "debug"; -$PSBINDIR = "$PSCONFDIR/$PSCONFIG.lin64/bin"; - -// END Local configuration +require "pstamp.php"; +require "submitted.php"; # this script sets up the environment to run IPP commands with current directory @@ -55,4 +42,5 @@ $diff_checked = ""; $list_checked = ""; +$getstatus_checked = ""; $pstamp_checked = ""; $get_checked = ""; @@ -102,5 +90,5 @@ if ($rvar_project == "gpc1") { $gpc1_selected = "selected"; - $require_class_id = 1; +// $require_class_id = 1; } else if ($rvar_project == "megacam-mops") { $mops_selected = "selected"; @@ -122,6 +110,6 @@ $diff_checked = "CHECKED"; } else { - // nothing checked default to By Exposure - $exp_checked = "CHECKED"; + // nothing checked default to By ID + $file_checked = "CHECKED"; } @@ -140,10 +128,10 @@ // is the center is specified in Pixels or sky coordinates -if ($rvar_center_type == "Sky") { +if ($rvar_center_type == "Pixels") { + $sky_checked=""; + $pix_checked="checked"; +} else { $sky_checked="checked"; $pix_checked=""; -} else { - $sky_checked=""; - $pix_checked="checked"; } @@ -168,16 +156,17 @@ } +$getstatus_checked = ""; +$pstamp_checked = ""; +$list_checked = ""; +$get_checked = ""; if ($rvar_cmd_mode == "Make Stamps") { $pstamp_checked = "checked"; - $list_checked = ""; - $get_checked = ""; } else if ($rvar_cmd_mode == "Get Images") { $get_checked = "checked"; - $list_checked = ""; - $pstamp_checked = ""; +} else if ($rvar_cmd_mode == "Get Status") { + $getstatus_checked = "checked"; } else { - $list_checked = "checked"; - $get_checked = ""; - $pstamp_checked = ""; + // default + $pstamp_checked = "checked"; } @@ -188,11 +177,10 @@ // How do we know whether or not this is the intial page load or not? -// Well, in that case rvar_img_type is not set so key off of that +// Well, the first time rvar_img_type is not set. So we key off of that. // TODO: find a better way to decide whether or not to run commands if ($rvar_img_type) { - $jobFinished = 0; - if ($request_id == 0) { + if (! $getstatus_checked) { try { $command_line = build_request_cmd(); @@ -201,19 +189,19 @@ if (! $list_checked) { // The only output from a successful run is the request_id + $request_name = trim(Array_pop($output_array)); $request_id = Array_pop($output_array); $last_request_id = $request_id; - setcookie("our_request_id", $request_id); - // echo "The request id is $request_id\n"; - if (count($output_array) != 0) { - throw new Exception("unexpected output returned by pstampwebrequest."); + if ($request_id && $request_name) { + addRequest($request_id, $request_name); + // setcookie("our_request_id", $request_id); + // echo "The request id is $request_id\n"; + $getstatus_checked = "checked"; + $pstamp_checked = ""; + } else { + // XXX: TODO print out the error + if (count($output_array) != 0) { + throw new Exception("unexpected output returned by pstampwebrequest."); + } } - - $jobRunning = getRequestStatus(); - if (!$jobRunning) { - // echo "1 getRequestStatus reuturned 0\n"; - $jobFinished = 1; - $request_id = 0; - } - } else { $last_request_id = 0; @@ -222,26 +210,8 @@ $error_line = $e->getMessage(); } - } else { - try { - // get the list of jobs for the request - // echo "calling getRequestStatus\n"; - $jobRunning = getRequestStatus(); - - if (!$jobRunning) { - // echo "2 getRequestStatus reuturned 0\n"; - $jobFinished = 1; - $request_id = 0; - } - - } catch (Exception $e) { - echo "Got Exception $request_id $e\n"; - $error_line = $e->getMessage(); - } - } - if ($last_request_id) { - // This doesn't work for get_image - listJobs($last_request_id, $jobFinished); - } -} + } +} + +// This is the end of the Logic function build_request_cmd() @@ -296,10 +266,10 @@ throw new Exception('RA and DEC must be specified.'); } - $cmd .= " -skycenter $rvar_RA $rvar_DEC"; + $cmd .= " --ra $rvar_RA --dec $rvar_DEC"; } else { if (! $rvar_X || ! $rvar_Y) { throw new Exception('X and Y must be specified.'); } - $cmd .= " -pixcenter $rvar_X $rvar_Y"; + $cmd .= " -pixcenter --x $rvar_X --y $rvar_Y"; } @@ -308,10 +278,10 @@ throw new Exception('dRA and dDEC must be specified.'); } - $cmd .= " -arcrange $rvar_dRA $rvar_dDEC"; + $cmd .= " --arcseconds --width $rvar_dRA --height $rvar_dDEC"; } else { if (! $rvar_W || ! $rvar_H) { throw new Exception('width and height must be specified.'); } - $cmd .= " -pixrange $rvar_W $rvar_H"; + $cmd .= " --width $rvar_W --height $rvar_H"; } } @@ -324,35 +294,37 @@ if ($exp_checked) { + if ($rvar_img_type == "stack") { + throw new Exception('Lookup by exposure name not supported for stack images.'); + } if (! $rvar_id ) { throw new Exception('Must set ID to the Exposure ID.'); } - $cmd .= " -byexp $rvar_img_type $rvar_id"; + $cmd .= " --req_type byexp --stage $rvar_img_type --id $rvar_id"; } else if ($file_checked) { if (! $rvar_id ) { throw new Exception('Must set ID to the exposure name.'); } - $cmd .= " -byid $rvar_img_type $rvar_id"; + $cmd .= " --req_type byid --stage $rvar_img_type --id $rvar_id"; } else if ($coord_checked) { - // $cmd .= " -bycoord $rvar_img_type"; - $coord_checked = ""; - $exp_checked = "checked"; - throw new Exception("Image selection by coordinate not implemented yet."); + $cmd .= " --req_type bycoord --stage $rvar_img_type"; + $coord_checked = "checked"; +// throw new Exception("Image selection by coordinate not implemented yet."); } else if ($diff_checked) { if (! $rvar_id ) { throw new Exception('Must set ID to Diff Image ID.'); } - $cmd .= " -bydiff $rvar_img_type $rvar_id"; - } - - if (($rvar_img_type == "raw") || ($rvar_img_type == "chip")) { - if ($require_class_id && ! $rvar_class_id ) { - throw new Exception("must specify Class ID with Image Type $rvar_img_type."); - } - if ((!$rvar_class_id) || ($rvar_class_id == "all")) { - $cmd .= " null"; - } else { - $cmd .= " $rvar_class_id"; - } - } + $cmd .= " --req_type bydiff --stage $rvar_img_type --id $rvar_id"; + } + +// XXX: don't need to require class_id anymore +// if (($rvar_img_type == "raw") || ($rvar_img_type == "chip")) { +// if (!$sky_checked && ($require_class_id && ! $rvar_class_id )) { +// throw new Exception("must specify Class ID with Image Type $rvar_img_type."); +// } + // leave off compoennt if we're looking up by coordinates. It breaks it + if (!$coord_checked && (($rvar_class_id) && ($rvar_class_id != "all"))) { + $cmd .= " --component $rvar_class_id"; + } +// } return escapeshellcmd($cmd); @@ -390,4 +362,5 @@ } +// This is no longer used function printURL($line) { @@ -442,119 +415,25 @@ echo ""; } - -function countRunningJobs() -{ - global $output_array; - - $runningJobs = 0; - $size = sizeof($output_array); - for ($i = 0; $i < $size; $i++) { - $elements = explode(" ", $output_array[$i]); - if (count($elements) == 3) { - $state = $elements[1]; - if ($state != "stop") { - $runningJobs++; - } - } else { - throw new Exception ("incorrect data in job status: $output_array[$i]"); - } - } - return $runningJobs; -} - -function listJobs($request_id, $jobFinished) -{ - global $WORKDIR; - global $SCRIPT; - - $command_line = "$SCRIPT pstamp_listjobs.pl $request_id"; - global $dbname; - global $dbserver; - if ($dbname) { - $command_line .= " --dbname $dbname --dbserver $dbserver"; - } - - run_command($command_line); - if ($jobFinished) { - global $outFileset; - global $dsroot; - $parse_error = "$WORKDIR/$request_id/parse_error.txt"; - #echo "reading $parse_error\n"; - // readfile( $parse_error ); - if (file_exists($parse_error)) { - $fhandle = fopen($parse_error, "r"); - if ($fhandle) { - $contents = fread($fhandle, 1024); - if ($contents) { - global $last_request_id; - global $error_line; - $error_line = "Request $last_request_id: $contents\n"; - } - } - } - } -} - -function getRequestStatus() -{ - global $request_id; - global $command_status; - global $output_array; - global $outFileset; - global $dbname; - global $dbserver; - global $SCRIPT; - - $command_line = "$SCRIPT pstamptool -listreq -req_id $request_id -simple"; - if ($dbname) { - $command_line .= " -dbname $dbname -dbserver $dbserver"; - } - // echo "Running $command_line\n"; - - run_command($command_line); - if ($command_status == 0) { - $size = sizeof($output_array); - $runningReq = 0; - for ($i = 0; $i < $size; $i++) { - $elements = explode(" ", $output_array[$i]); - if (count($elements) >= 4) { - $state = $elements[2]; - $outFileset = $elements[4]; - if ($state != "stop") { - $runningReq++; - } - } else { - throw new Exception ("incorrect data in job status: $output_array[$i]"); - } - } - return $runningReq; - } else { - return 0; - } -} - ?> - + + - - Postage Stamp Request Form - '; - - } - ?> - + + + Postage Stamp Request Form (prototype) + + + + +

+Postage Stamp Request Form +

+ + +
- - - -

-Postage Stamp Request -

- @@ -586,8 +465,9 @@  Select Images By:    + >Database ID +  >Exposure Name   - >Database ID -  + >Coordinates   @@ -625,5 +505,5 @@ echo "Chip ID:"; } else { - echo "Class ID:"; + echo "Component:"; } ?> @@ -685,4 +565,5 @@   > +   " @@ -691,4 +572,5 @@   +   " @@ -742,19 +624,16 @@ - Mode:   + >Get Status >Make Stamps - >Get Images + >Get Bundles + - - - Request Id: $request_id"; - endif; ?> + // echo "Request Id: $request_id"; +?> @@ -766,6 +645,14 @@ + + + + + + Last Command @@ -794,8 +681,11 @@ - - +size of output array is $size\n"; @@ -818,4 +708,5 @@ } } +} // end if if(0) ?>
ResultsRequest Results
@@ -845,4 +736,8 @@ + + @@ -861,6 +756,5 @@ ?> + - - Index: /branches/eam_branches/ipp-20100823/tools/czarclean.pl =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czarclean.pl (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/czarclean.pl (revision 29124) @@ -0,0 +1,62 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +# local classes +use czartool::CzarDb; + +my $czarDbName = undef; +my $begin = undef; +my $end = undef; +my $interval = undef; +my $verbose = undef; + +GetOptions ( + "dbname|d=s" => \$czarDbName, + "interval|i=s" => \$interval, + "begin|b=s" => \$begin, + "end|e=s" => \$end, + "verbose|v" => \$verbose, + ); + +print "\n*******************************************************************************\n"; +print "* \n"; +my $quit = 0; + +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} +if (!$begin) { + $quit=1; + print "* REQUIRED: choose a begin date -b (default=7am this morning)\n"; +} +if (!$end) { + $end=$begin; + print "* OPTIONAL: choose an end date -e (default=$end)\n"; +} +if (!$czarDbName) { + $czarDbName = "czardb"; + print "* OPTIONAL: choose czar Db name -d (default=$czarDbName\n"; +} +if (!$interval) { + $interval = "30 MINUTE"; + print "* OPTIONAL: choose time interval -i <'1 hour'|'2 hour'|etc> (default=$interval)\n"; +} +print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + + +my $save_temps = 0; + + +my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg + +my $labels = undef; + + +$czarDb->cleanupDateRange($begin, $end, $interval); + Index: /branches/eam_branches/ipp-20100823/tools/czarplot.pl =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czarplot.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/tools/czarplot.pl (revision 29124) @@ -8,5 +8,5 @@ use czartool::CzarDb; -use czartool::Czarplot; +use czartool::Plotter; my $czarDbName = "czardb"; @@ -23,4 +23,5 @@ my $nebulous = undef; my $savingToFile = undef; +my $log = undef; GetOptions ( @@ -36,19 +37,30 @@ "timeseries|t" => \$timeSeries, "verbose|v" => \$verbose, + "log|g" => \$log, ); -print "\n"; +print "\n*******************************************************************************\n"; +print "* \n"; +my $quit = 0; +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} if (!$histogram) { print "* OPTIONAL: plot histogram -h (default=off)\n";} if (!$timeSeries) { print "* OPTIONAL: plot timeseries -t (default=on)\n";} +if (!$log) { + $log = 0; + print "* OPTIONAL: use log plots -g (default=$log\n";} if (!$nebulous) { print "* OPTIONAL: plot nebulous disk space -n (default=off)\n";} if (!$label) { - print "* OPTIONAL: choose a label -l (default='all_stdscience_labels')\n";} + $label = "all_stdscience_labels"; + print "* OPTIONAL: choose a label -l (default=$label)\n";} if (!$stage) { print "* OPTIONAL: choose a stage -s (default=none)\n";} if (!$interval) { - print "* OPTIONAL: choose time interval in past -i <'1 hour'|'1 day'|etc> (default=interval since 7am this morning)\n";} + print "* OPTIONAL: choose time interval in past -i <'1 hour'|'1 day'|etc> (default=none\n";} if (!$begin) { print "* OPTIONAL: choose a begin time -b (default=7am this morning)\n";} @@ -56,10 +68,11 @@ print "* OPTIONAL: choose an end time -e (default=now)\n";} if (!$path) { - print "* OPTIONAL: choose an output location -o (default=none)\n";} + print "* OPTIONAL: choose an output location -o (default=none)\n"; +} +print "*\n*******************************************************************************\n"; -print "\n"; +if ($quit) { exit; } # default values -if (!$label) {$label = "all_stdscience_labels";} if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;} if (!$verbose) {$verbose = 0;} @@ -71,7 +84,7 @@ $czarDb->setDateFormat("%Y%m%d-%H%i%s"); -my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps); -my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); +# GENE PLOTS my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png size 1280,960 font \"/usr/share/fonts/corefonts/arial.ttf\" 12" : "X11", $path, $save_temps); +my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps); # sort out times @@ -79,12 +92,12 @@ if (!$begin) { if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);} - else {$begin = strftime('%Y-%m-%d 07:00',localtime);} + else {$begin = strftime('%Y-%m-%d 06:35',localtime);} } my $diskUsage = 1; # TODO -if (!$nebulous && $timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);} -if ($histogram) {$czarplot->createHistogram($label, $begin, $end);} -if ($nebulous && $timeSeries) {$czarplot->plotStorageTimeSeries($begin, $end);} -elsif ($nebulous) {$czarplot->plotDiskUsageHistogram();} +if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log);} +if ($histogram) {$plotter->createHistogram($label, $begin, $end);} +if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);} +elsif ($nebulous) {$plotter->plotDiskUsageHistogram();} Index: /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/czartool/Burntool.pm (revision 29124) @@ -0,0 +1,68 @@ +#!/usr/bin/perl -w +package czartool::Burntool; + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + my $class = shift; + my $self = {}; + $self->{_verbose} = 0; + + bless $self, $class; + return $self; +} + +########################################################################### +# +# Get pending and processed for this label +# +########################################################################### +sub getPendingAndProcessed { + my ($self, $label, $pending, $processed) = @_; + + ${$pending} = ${$processed} = 0; + + my $target = undef; + + if ($label =~ m/(.*)\.nightlyscience/) { + + $target = $1; + } + else {return 0;} + + # get today's date + my ($day, $month, $year) = (localtime)[3,4,5]; + my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day); + + if ($self->{_verbose}) {print "Checking $target for $self->{_today}\n";} + my $command = "automate_stacks.pl --check_chips --burntool_stats --this_target_only $target --date $today"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $self->{_verbose}); + + if (!$success) {return 0;} + + my $line; + foreach $line (@{$stdout_buf}) { + + chomp($line); + if($line =~ m/([0-9]+)\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)/) { + if ($self->{_verbose}) {print "Output = $1 $2 $3 $4\n";} + ${$pending} = ($2 - $3)/60; + ${$processed} = $3/60; + return 1; + } + } + + return 0; +} +1; + + Index: /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm (revision 29123) +++ /branches/eam_branches/ipp-20100823/tools/czartool/CzarDb.pm (revision 29124) @@ -1,3 +1,3 @@ -#!/usr/bin/perl i-w +#!/usr/bin/perl -w package czartool::CzarDb; @@ -6,5 +6,5 @@ use strict; -my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere +my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere use base 'czartool::MySQLDb'; @@ -119,4 +119,28 @@ $query->execute; +} + +########################################################################### +# +# Updates nightlyscience table +# +########################################################################### +sub updateNightlyScience { + my ($self, $status) = @_; + + my $query = $self->{_db}->prepare(<execute; + + $query = $self->{_db}->prepare(<execute; } @@ -218,8 +242,9 @@ ########################################################################### sub createTimeSeriesData { - my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_; - - my $dataFile = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat"; - open (GNUDAT, ">$dataFile"); + my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_; + + ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat"; + + open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n"; my $query = $self->{_db}->prepare(<execute) {return undef;} + if (!$query->execute) {return 0;} my $minProcessed = undef; - - ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); - - if (!defined $minProcessed) {return undef;} + my ($_maxY, $_minY, $_timeDiff); + ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array(); + + if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;} + if ($_minY < ${$minY}) {${$minY} = $_minY;} + if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;} + + if (!defined $minProcessed) {return 0;} $query = $self->{_db}->prepare(<fetchrow_array()) { - my ($timestamp, $pending, $faults, $processed) = @row; - # print "'$timestamp' '$pending' '$faults' '$processed'\n"; + + if ($isLog) { + + $processed = log($processed + 1); + $pending = log($pending + 1); + $faults = log($faults + 1); + } + print GNUDAT "$timestamp $pending $faults $processed\n"; } - close(GNUDAT); - - return $dataFile; + + close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n"; + + return 1; } @@ -491,4 +527,117 @@ } + +########################################################################### +# +# Deletes all but one row per interval from all stage tables for all labels between the two dates +# +########################################################################### +sub cleanupDateRange { + my ($self, $startDay, $endDay, $interval) = @_; + + my $thisDay = $startDay; + my $quit = 0; + while(!$quit) { + + if (!$self->isBefore($thisDay, $endDay)) { + + $quit = 1; + } + + print "* Running cleanup for $thisDay with an interval of $interval\n"; + $self->cleanupADay($thisDay, $interval); + $thisDay = $self->addInterval($thisDay, "1 DAY"); + + } +} +########################################################################### +# +# Deletes all but one row per interval from all stage tables for all labels between the provided day +# +########################################################################### +sub cleanupADay { + my ($self, $startDay, $interval) = @_; + + my $endDay = $self->addInterval($startDay, "1 DAY"); + + my $labels = undef; + my $fromTime = $startDay; + my $toTime = undef; + my $totalDeleted = undef; + my $quit = 0; + while(!$quit) { + + $toTime = $self->addInterval($fromTime, $interval); + if (!$self->isBefore($toTime, $endDay)) { + + $toTime = $endDay; + $quit = 1; + } + + my $stage = undef; + $totalDeleted = 0; + foreach $stage (@stages) { + + if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;} + + my $label = undef; + my $row = undef; + foreach $row ( @{$labels} ) { + my ($label) = @{$row}; + + my $query = $self->{_db}->prepare(< '$fromTime' + AND timestamp <= '$toTime' + AND label = '$label' +SQL + + $query->execute; + + my $toDelete = scalar $query->fetchrow_array() - 1; + if ($toDelete < 1) {next;} + + $query = $self->{_db}->prepare(< '$fromTime' + AND timestamp <= '$toTime' + AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete +SQL + $query->execute; + + $totalDeleted += $toDelete; + } + } + print " * Deleted $totalDeleted between $fromTime and $toTime\n"; + $fromTime = $toTime; + } +} + +########################################################################### +# +# Returns an array of labels present during the provided time frame +# +########################################################################### +sub getLabelsInThisTimePeriod { + my ($self, $stage, $fromTime, $toTime, $labels) = @_; + + my $query = $self->{_db}->prepare(< '$fromTime' + AND timestamp <= '$toTime'; +SQL + + if (!$query->execute) { + + return 0; + } + + ${$labels} = $query->fetchall_arrayref(); + + return 1; +} + ########################################################################### # @@ -500,7 +649,6 @@ my $currentRevision = -1; - my $latestRevision = 10; - - while ($currentRevision != $latestRevision) { + + while (1) { $currentRevision = $self->getRevision(); @@ -517,4 +665,7 @@ elsif ($currentRevision == 8) {$self->createRevision_9();} elsif ($currentRevision == 9) {$self->createRevision_10();} + elsif ($currentRevision == 10) {$self->createRevision_11();} + elsif ($currentRevision == 11) {$self->createRevision_12();} + else {last;} } } @@ -841,4 +992,56 @@ } +####################################################################################### +# +# Create revision 11 of the database +# +####################################################################################### +sub createRevision_11 { + my ($self) = @_; + + print "* Creating revision 11 of '$self->{_dbName}'\n"; + + # same shape as other stage tables to enable easy update + my $query = $self->{_db}->prepare(<execute; + $query = $self->{_db}->prepare(<execute; + + $self->setRevision(11); +} + +####################################################################################### +# +# Create revision 12 of the database +# +####################################################################################### +sub createRevision_12 { + my ($self) = @_; + + print "* Creating revision 12 of '$self->{_dbName}'\n"; + + # same shape as other stage tables to enable easy update + my $query = $self->{_db}->prepare(<execute; + + $self->setRevision(12); +} + 1; Index: anches/eam_branches/ipp-20100823/tools/czartool/Czarplot.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/Czarplot.pm (revision 29123) +++ (revision ) @@ -1,393 +1,0 @@ -#!/usr/bin/perl -w -package czartool::Czarplot; - -use warnings; -use strict; - -my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); - - -########################################################################### -# -# Constructor -# -########################################################################### -sub new { - my $class = shift; - my $self = { - _czarDb => shift, - _dateFormat => shift, - _outputFormat => shift, - _outputPath => shift, - _save_temps => shift, - }; - - bless $self, $class; - return $self; -} - -########################################################################### -# -# Generates a suitable filename for this plot -# -########################################################################### -sub createImageFileName { - my ($self, $label, $stage, $suffix) = @_; - - my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; - my $stagePart = $stage ? $stage : "all_stages"; - return $prefix . "/czarplot_" . $label . "_" . $stagePart . "_$suffix.png"; -} - -########################################################################### -# -# Plots a time series for all stages for this label -# -########################################################################### -sub createTimeSeries { - my ($self, $label, $selectedStage, $beginTime, $endTime) = @_; - - my $outputFile = createImageFileName($self, $label, $selectedStage, "t"); - - my ($minX, $maxX, $minY, $maxY, $timeDiff); - $minX = 999999999; - $maxX = -9999999999; - $minY = 999999999; - $maxY = -99999999; - - my $stages = undef; - - if (!$selectedStage) {$stages = \@allStages;} - else {$stages = ["$selectedStage"]}; - - my %gnuplotFiles; - my $stage = undef; - my $gnuplotFile = undef; - foreach $stage (@{$stages}) { - - $gnuplotFile = $self->{_czarDb}->createTimeSeriesData($label, $stage, $beginTime, $endTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff); - if ($gnuplotFile) {$gnuplotFiles{$stage} = $gnuplotFile;} - - } - - my $numOfPlots = keys(%gnuplotFiles); - - if ($numOfPlots == 0 ) { - - print "Warning: No plots could be generated for '$label' during time period '$beginTime', '$endTime'\n"; - return; - } - - if ($timeDiff == 0) { - - print "WARNING: Zero time difference for '$label' during time period '$beginTime', '$endTime'\n"; - return; - } - - plotTimeSeries($self, \%gnuplotFiles, $outputFile, $label, $beginTime, $endTime, $maxX, $minX, $maxY, $minY, $timeDiff); -} - -########################################################################### -# -# Plots a histogram of stuff processed in provided interval and label for all stages -# -########################################################################### -sub createHistogram { - my ($self, $label, $beginTime, $endTime) = @_; - - my $outputFile = createImageFileName($self, $label, undef, "h"); - - my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table - open (GNUDAT, ">$inputFile"); - - my ($processed, $pending, $faults); - my $stage = undef; - foreach $stage (@allStages) { - - $self->{_czarDb}->countProcessedPendingAndFaults($label, $stage, $beginTime, $endTime, \$processed, \$pending, \$faults); - print GNUDAT "$stage $processed, $pending, $faults\n"; - } - - close(GNUDAT); - - plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime); -} - - -########################################################################### -# -# Sets the output path -# -########################################################################### -sub setOutputFormat { - my ($self, $outputFormat) = @_; - $self->{_outputFormat} = $outputFormat if defined($outputFormat); -} - -########################################################################### -# -# Sets the output type -# -########################################################################### -sub setOutputPath { - my ($self, $outputPath) = @_; - $self->{_outputPath} = $outputPath if defined($outputPath); -} - -########################################################################### -# -# Sets the date format to be used -# -########################################################################### -sub setDateFormat { - my ($self, $dateFormat) = @_; - $self->{_dateFormat} = $dateFormat if defined($dateFormat); - return $self->{_dateFormat}; -} - -########################################################################### -# -# Sorts out plotting max/min x/y -# -########################################################################### -sub sortOutMaxMinY { - my ($self, $minY, $maxY) = @_; - - my $tmp; - if (${$maxY} < ${$minY}) { - - $tmp = ${$maxY}; - ${$maxY} = ${$minY}; - ${$minY} = $tmp; - } - - my $border = (${$maxY} - ${$minY})/10; - - ${$maxY} += $border; - ${$minY} -= $border ; - if (${$maxY} == 0) {${$maxY} = 1;} - if (${$minY} == 0) {${$minY} = -1;} -} - - -########################################################################### -# -# Figures out suitable spacing for time tics on time-series pliots -# -########################################################################### -sub getTimeSpacing { - my ($self, $timeDiff, $divX, $timeFormat) = @_; - - # if less than a couple of hour's data plotted, show 30 mins tics - if ($timeDiff < 7200) { - ${$timeFormat} = "%H:%M"; - ${$divX} = 1800; - } - # if less than half a day's data plotted, show hourly tics - elsif ($timeDiff < 43200) { - ${$timeFormat} = "%H:%M"; - ${$divX} = 3600; - } - # if less than one day's data plotted, show 2 hourly tics - elsif ($timeDiff < 86400) { - ${$timeFormat} = "%H:%M"; - ${$divX} = 7200; - } - # if less than 1 week's data is data plotted, show daily tics - elsif ($timeDiff < 604800) { - - ${$timeFormat} = "%m/%d"; - ${$divX} = 86400; - } - else { - - ${$timeFormat} = "%m/%d"; - ${$divX} = 172800; - } -} - -########################################################################### -# -# Plots a time-series of pending/faults -# -########################################################################### -sub plotTimeSeries { - my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff) = @_; - - $self->sortOutMaxMinY(\$minY, \$maxY); - - my $timeFormat = undef; - my $divX = undef; - - $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); - - my $numOfPlots = keys %$gnuplotFiles; - my $title = undef; - - # sort out plot title - if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}} - else {$title = "'All stages'"} - - $title .= " for '$label', '$fromTime' to '$toTime'"; - - open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; - use FileHandle; - GP->autoflush(1); - - print GP - "set term $self->{_outputFormat};" . - "set output \"$outputFile\";" . - "set title \"$title\";" . - "set key left top;" . - "set xdata time;" . - "set timefmt \"$self->{_dateFormat}\";" . - # "set yrange [\"$minY\":\"$maxY\"];" . - "set xrange [\"$minX\":\"$maxX\"];" . - "set format x \"$timeFormat\";" . - "set xtics \"$minX\", $divX, \"$maxX\";" . - "set grid xtics;" . - "set xlabel \"Time\";" . - "set ylabel \"Exposures\";" . - "plot "; - - my $firstIn = 1; - foreach my $stage (keys %$gnuplotFiles) { - if (!$firstIn) {print GP ",";} - - # for single stage plots, show pending/processed/faults - if ($numOfPlots == 1) { - - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2,"; - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4,"; - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7"; - } - # when plotting multiple stages, show only processed - else { - print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines"; - } - $firstIn = 0; - } - - print GP "\n"; - close GP; -} - -########################################################################### -# -# Plots a time-series of cluster storage -# -########################################################################### -sub plotStorageTimeSeries { - my ($self, $fromTime, $toTime) = @_; - - my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; - my $outputFile = "$prefix/czarplot_cluster.png"; - my ($minX, $maxX, $minY, $maxY, $timeDiff); - - my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff); - - $self->sortOutMaxMinY(\$minY, \$maxY); - - my $timeFormat = undef; - my $divX = undef; - - $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); - - open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; - use FileHandle; - GP->autoflush(1); - - print GP - "set term $self->{_outputFormat};" . - "set output \"$outputFile\";" . - "set title \"Total available cluster space over time\";" . - "set key left top;" . - "set xdata time;" . - "set timefmt \"$self->{_dateFormat}\";" . - "set xrange [\"$minX\":\"$maxX\"];" . - "set yrange [\"$minY\":\"$maxY\"];" . - "set format x \"$timeFormat\";" . - "set xtics \"$minX\", $divX, \"$maxX\";" . - "set grid xtics;" . - "set xlabel \"Time\";" . - "set ylabel \"Available (TB)\";" . - "plot " . -# "'$gnuplotFile' using 1:2 title \"Total\" with lines lt 1," . - "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2\n"; - - print GP "\n"; - close GP; -} - -########################################################################### -# -# Plots a histogram of processed stuff -# -########################################################################### -sub plotHistogram { - my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_; - - open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; - use FileHandle; - GP->autoflush(1); - - print GP - "set term $self->{_outputFormat};" . - "set output \"$outputFile\";" . - "set title \"'$label', '$fromTime' to '$toTime'\";" . - "set grid;" . - "set boxwidth;" . - "set style data histogram;" . - "set style histogram cluster gap 3;" . - "set style fill solid border -1;" . - "set bmargin 5;" . - "set ylabel \"Exposures\";" . - "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 4, '' using 4 title \"Faults\" lt 7;" . - "\n"; - - close GP; -} - -########################################################################### -# -# Plots disk usage across cluster as a stacked histogram -# -########################################################################### -sub plotDiskUsageHistogram { - my ($self) = @_; - - my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this - my $outputFile = "$prefix/czarplot_hosts_space.png"; - my $limit = 97.0; - my $inputFile = $self->{_czarDb}->createHostsData($limit); - - my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage(); - - my $totalPercent = sprintf("%.1f%%", $totalUsed); - - open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; - use FileHandle; - GP->autoflush(1); - - print GP - "set term $self->{_outputFormat};" . - "set output \"$outputFile\";" . - "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . - "set style fill solid 1.00 border -1;" . - "set style histogram rowstacked;" . - "set style data histograms;" . - "set ylabel \"Space (TB)\";" . - "set xtic rotate by -90 scale 0;" . - "plot '$inputFile' " . - "using 2:xtic(1) t \"Used\" lt 7," . - "'' using 3 t \"Over $limit% used\" lt 1," . - "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," . - "'' using 5 t \"Free\" lt 2," . - "'' using 6 notitle fs solid 0.50 lt -1" . - ";" . - - "\n"; - - close GP; -} -1; Index: /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm (revision 29123) +++ /branches/eam_branches/ipp-20100823/tools/czartool/MySQLDb.pm (revision 29124) @@ -64,4 +64,20 @@ return $self->{_dbHost}; } + +########################################################################### +# +# Adds the provided interval to the provided time +# +########################################################################### +sub addInterval { + my ($self, $time, $interval) = @_; + + my $query = $self->{_db}->prepare(<execute; + +return scalar $query->fetchrow_array(); +} ########################################################################### Index: /branches/eam_branches/ipp-20100823/tools/czartool/Pantasks.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/Pantasks.pm (revision 29123) +++ /branches/eam_branches/ipp-20100823/tools/czartool/Pantasks.pm (revision 29124) @@ -73,4 +73,32 @@ } return 1; +} + +########################################################################### +# +# Gets nightlyscience status for today +# +########################################################################### +sub getNightlyScienceStatus { + my ($self, $status) = @_; + + my @labels; + + my ($day, $month, $year) = (localtime)[3,4,5]; + my $today = sprintf("%04d-%02d-%02d", $year+1900, $month+1, $day); + + my @cmdOut = `echo "ns.show.dates;quit" | pantasks_client -c ~ipp/stdscience/ptolemy.rc 2>&1`; + my $line; + foreach $line (@cmdOut) { + + chomp($line); + if (!$self->outputOk($line)) {return 0;} + if ($line =~ m/$today\s+(.+)/) { + ${$status} = $1; + return 1; + } + } + + return 0; } Index: /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm =================================================================== --- /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/czartool/Plotter.pm (revision 29124) @@ -0,0 +1,425 @@ +#!/usr/bin/perl -w +package czartool::Plotter; + +use warnings; +use strict; + +my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); + + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + my $class = shift; + my $self = { + _czarDb => shift, + _dateFormat => shift, + _outputFormat => shift, + _outputPath => shift, + _save_temps => shift, + }; + + bless $self, $class; + return $self; +} + +########################################################################### +# +# Generates a suitable filename for this plot +# +########################################################################### +sub createImageFileName { + my ($self, $label, $stage, $suffix, $isLog) = @_; + + my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; + my $stagePart = $stage ? $stage : "all_stages"; + my $type = $isLog ? "log" : "linear"; + return $prefix . "/czarplot_" . $type . "_" . $label . "_" . $stagePart . "_$suffix.png"; +} + +########################################################################### +# +# Plots a time series for all stages for this label, bith log and linear +# +########################################################################### +sub createLogAndLinearTimeSeries { + my ($self, $label, $selectedStage, $beginTime, $endTime) = @_; + + $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 1); + $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 0); +} + +########################################################################### +# +# Plots a time series for all stages for this label +# +########################################################################### +sub createTimeSeries { + my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog) = @_; + + my ($minX, $maxX, $minY, $maxY, $timeDiff); + $minX = 999999999; + $maxX = -9999999999; + $minY = 999999999; + $maxY = -99999999; + $timeDiff = 0; + + my $stages = undef; + + if (!$selectedStage) {$stages = \@allStages;} + else {$stages = ["$selectedStage"]}; + + my %gnuplotFiles; + my $stage = undef; + my $gnuplotFile = undef; + foreach $stage (@{$stages}) { + + if($self->{_czarDb}->createTimeSeriesData( + $label, + $stage, + $beginTime, + $endTime, + \$minX, + \$maxX, + \$minY, + \$maxY, + \$timeDiff, + \$gnuplotFile, + $isLog)) { + + $gnuplotFiles{$stage} = $gnuplotFile; + } + } + + my $numOfPlots = keys(%gnuplotFiles); + + if ($numOfPlots == 0 ) { + + print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n"; + return; + } + + if ($timeDiff == 0) { + + print "WARNING: Zero time difference for '$label' during time period '$beginTime', '$endTime'\n"; + return; + } + + my $outputFile = createImageFileName($self, $label, $selectedStage, "t", $isLog); + $self->plotTimeSeries( + \%gnuplotFiles, + $outputFile, + $label, + $beginTime, + $endTime, + $maxX, + $minX, + $maxY, + $minY, + $timeDiff, + $isLog); +} + +########################################################################### +# +# Plots a histogram of stuff processed in provided interval and label for all stages +# +########################################################################### +sub createHistogram { + my ($self, $label, $beginTime, $endTime) = @_; + + my $outputFile = createImageFileName($self, $label, undef, "h", 0); + + my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table + open (GNUDAT, ">$inputFile"); + + my ($processed, $pending, $faults); + my $stage = undef; + my $pendingMinusFaults = undef; + my $maxY = 0; + my $sum; + foreach $stage (@allStages) { + + $self->{_czarDb}->countProcessedPendingAndFaults( + $label, + $stage, + $beginTime, + $endTime, + \$processed, + \$pending, + \$faults); + + $pendingMinusFaults = $pending - $faults; + print GNUDAT "$stage $faults $processed, $pendingMinusFaults\n"; + my $sum = $faults + $processed + $pendingMinusFaults; + if ($sum > $maxY) {$maxY = $sum;} + } + + close(GNUDAT); + + plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime, $maxY); + unlink($inputFile); +} + + +########################################################################### +# +# Sets the output path +# +########################################################################### +sub setOutputFormat { + my ($self, $outputFormat) = @_; + $self->{_outputFormat} = $outputFormat if defined($outputFormat); +} + +########################################################################### +# +# Sets the output type +# +########################################################################### +sub setOutputPath { + my ($self, $outputPath) = @_; + $self->{_outputPath} = $outputPath if defined($outputPath); +} + +########################################################################### +# +# Sets the date format to be used +# +########################################################################### +sub setDateFormat { + my ($self, $dateFormat) = @_; + $self->{_dateFormat} = $dateFormat if defined($dateFormat); + return $self->{_dateFormat}; +} + +########################################################################### +# +# Figures out suitable spacing for time tics on time-series pliots +# +########################################################################### +sub getTimeSpacing { + my ($self, $timeDiff, $divX, $timeFormat) = @_; + + # if less than a couple of hour's data plotted, show 30 mins tics + if ($timeDiff < 7200) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 1800; + } + # if less than half a day's data plotted, show hourly tics + elsif ($timeDiff < 43200) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 3600; + } + # if less than one day's data plotted, show 2 hourly tics + elsif ($timeDiff < 86400) { + ${$timeFormat} = "%H:%M"; + ${$divX} = 7200; + } + # if less than 1 week's data is data plotted, show daily tics + elsif ($timeDiff < 604800) { + + ${$timeFormat} = "%m/%d"; + ${$divX} = 86400; + } + else { + + ${$timeFormat} = "%m/%d"; + ${$divX} = 172800; + } +} + +########################################################################### +# +# Plots a time-series of pending/faults +# +########################################################################### +sub plotTimeSeries { + my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, $isLog) = @_; + + my $timeFormat = undef; + my $divX = undef; + my $yTitle = undef; + if ($isLog) {$yTitle = "Log( numExposures )";} + else {$yTitle = "numExposures";} + + $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); + + my $numOfPlots = keys %$gnuplotFiles; + my $title = undef; + + # sort out plot title + if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}} + else {$title = "'All stages'"} + + $title .= " for '$label', '$fromTime' to '$toTime'"; + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"$title\";" . + "set key left top;" . + "set xdata time;" . + "set timefmt \"$self->{_dateFormat}\";" . + "set format x \"$timeFormat\";" . + "set xtics \"$minX\", $divX, \"$maxX\";" . + "set grid xtics;" . + "set xlabel \"Time\";" . + "set ylabel \"$yTitle\";" . + "plot "; + + my $firstIn = 1; + foreach my $stage (keys %$gnuplotFiles) { + if (!$firstIn) {print GP ",";} + + # for single stage plots, show pending/processed/faults + if ($numOfPlots == 1) { + + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,"; + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,"; + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7 lw 2"; + } + # when plotting multiple stages, show only processed + else { + + print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines lw 2"; + } + $firstIn = 0; + } + + print GP "\n"; + close GP; + + # now delete temp dat files + foreach my $stage (keys %$gnuplotFiles) { + + unlink($gnuplotFiles->{$stage}); + } +} + +########################################################################### +# +# Plots a time-series of cluster storage +# +########################################################################### +sub plotStorageTimeSeries { + my ($self, $fromTime, $toTime) = @_; + + my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; + my $outputFile = "$prefix/czarplot_cluster.png"; + my ($minX, $maxX, $minY, $maxY, $timeDiff); + + my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff); + + my $timeFormat = undef; + my $divX = undef; + + $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat); + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"Total available cluster space over time\";" . + "set key left top;" . + "set xdata time;" . + "set timefmt \"$self->{_dateFormat}\";" . + "set format x \"$timeFormat\";" . + "set xtics \"$minX\", $divX, \"$maxX\";" . + "set grid xtics;" . + "set xlabel \"Time\";" . + "set ylabel \"Available (TB)\";" . + "plot " . + "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2 lw 2\n"; + + print GP "\n"; + close GP; + unlink($gnuplotFile); +} + +########################################################################### +# +# Plots a histogram of processed stuff +# +########################################################################### +sub plotHistogram { + my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime, $maxY) = @_; + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + if ($maxY == 0) {$maxY = 1;} + else {$maxY = $maxY*1.1;} + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"'$label', '$fromTime' to '$toTime'\";" . + "set grid;" . + "set boxwidth;" . + "set yrange [\"0\":\"$maxY\"];" . + "set style data histogram;" . + "set style histogram rowstacked;" . + "set style fill solid border -1;" . + "set ylabel \"Exposures\";" . + "set boxwidth 0.75;" . + "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . + "\n"; + + close GP; +} + +########################################################################### +# +# Plots disk usage across cluster as a stacked histogram +# +########################################################################### +sub plotDiskUsageHistogram { + my ($self) = @_; + + my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this + my $outputFile = "$prefix/czarplot_hosts_space.png"; + my $limit = 97.0; + my $inputFile = $self->{_czarDb}->createHostsData($limit); + + my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage(); + + my $totalPercent = sprintf("%.1f%%", $totalUsed); + + open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; + use FileHandle; + GP->autoflush(1); + + print GP + "set term $self->{_outputFormat};" . + "set output \"$outputFile\";" . + "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . + "set style fill solid 1.00 border -1;" . + "set style histogram rowstacked;" . + "set style data histograms;" . + "set ylabel \"Space (TB)\";" . + "set xtic rotate by -90 scale 0;" . + "plot '$inputFile' " . + "using 2:xtic(1) t \"Used\" lt 7," . + "'' using 3 t \"Over $limit% used\" lt 1," . + "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," . + "'' using 5 t \"Free\" lt 2," . + "'' using 6 notitle fs solid 0.50 lt -1" . + ";" . + + "\n"; + + close GP; +} +1; Index: /branches/eam_branches/ipp-20100823/tools/neb-grep =================================================================== --- /branches/eam_branches/ipp-20100823/tools/neb-grep (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/neb-grep (revision 29124) @@ -0,0 +1,90 @@ +#!/usr/bin/perl -w + +$debug = "/dev/null"; +open DEBUG, ">$debug" or die "Can't open $debug"; +# Note: +# Debug messages are prepended with the "DEBUG: " pattern +# Therefore: +# neb-grep -v GREP_PATTERN FILE_PATTERN | grep -v ^DEBUG +# is equivalent to: +# neb-grep GREP_PATTERN FILE_PATTERN + +# +# We need at least two arguments +# +if (@ARGV < 2) { + $usage = "Usage: neb-grep [--verbose|-v] [--server ] [GREP OPTIONS] \n"; + $usage .= " See grep man page for and [GREP OPTIONS]\n"; + $usage .= " is a SQL-like pattern (file_pattern search is based on neb-ls --path)\n"; + $usage .= " http://nebulous.mhpcc.ipp.ifa.hawaii.edu/nebulous is default value\n"; + $usage .= " [-v|--verbose] show details\n"; + $usage .= "\n"; + $usage .= " Example(s):\n"; + $usage .= " neb-grep \"my_die\" gpc1/ThreePi.nt/2010/08/24/o5432g0503o.212622/o5432g0503o.212622.ch.126887.%.log\n"; + $usage .= " neb-grep -i \"MY_DIE\" gpc1/ThreePi.nt/2010/08/24/o5432g0503o.212622/o5432g0503o.212622.ch.126887.%.log\n"; + &bye(2, $usage); +} + +# +# Parse arguments +# +$args = ""; +while (@ARGV > 1) { + if ($ARGV[0] =~ /--path/) { + #Forget it since it's always added + shift; + } elsif ( $ARGV[0] =~ /--server/ ) { + shift; + $server = $ARGV[0]; + shift; + } elsif ( ($ARGV[0] =~ /--verbose/) || ($ARGV[0] =~ /-v/) ){ + close DEBUG; + $debug = "/dev/stdout"; + open DEBUG, ">$debug" or die "Can't open $debug"; + shift; + } else { + $args .= $ARGV[0]." "; + shift; + } +} +$file_pattern=$ARGV[0]; +chop $args; +print DEBUG "DEBUG: file_pattern = [$file_pattern]\n"; +print DEBUG "DEBUG: grep arguments = [$args]\n"; + +# Define server with the default value if needed +if (!(defined $server)) { + $server = "http://nebulous.mhpcc.ipp.ifa.hawaii.edu/nebulous"; +} + +# Get the filenames from neb-ls command +print DEBUG "DEBUG: Executing command: neb-ls --path --server $server --path $file_pattern 2>&1\n"; +@nebitems = `neb-ls --path --server $server --path $file_pattern 2>&1`; + +# Deal with non-matching file patterns +if ( (@nebitems == 0) || ($nebitems[0] =~ /no instances/) ) { + &bye(1, "no instances"); +} + +# Deal with matching file patterns by running the "grep" command +for $filename (@nebitems) { + chomp $filename; + print DEBUG "DEBUG: Executing grep $args $filename\n"; + $output = `grep $args $filename`; + print DEBUG "DEBUG: Output is [$output]\n"; + if (!($output =~ /^$/)) { + print $filename, ": ", $output; + } +} +&bye(0, ""); + +sub bye { + my $status = shift; + my $message = shift; + close DEBUG; + unless ($message =~ /^$/) { + print STDERR $message,"\n"; + } + exit $status; +} + Index: /branches/eam_branches/ipp-20100823/tools/roboczar.pl =================================================================== --- /branches/eam_branches/ipp-20100823/tools/roboczar.pl (revision 29123) +++ /branches/eam_branches/ipp-20100823/tools/roboczar.pl (revision 29124) @@ -11,5 +11,6 @@ use czartool::Pantasks; use czartool::Nebulous; -use czartool::Czarplot; +use czartool::Plotter; +use czartool::Burntool; my $period = 60; @@ -26,8 +27,10 @@ my $nebulous = new czartool::Nebulous($czarDb); my $pantasks = new czartool::Pantasks(); -my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path +my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path +my $burntool = new czartool::Burntool(); + $czarDb->setDateFormat("%Y%m%d-%H%i%s"); -my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); +my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); @@ -101,9 +104,10 @@ my $priority = undef; my $newState = undef; + my $nsStatus = undef; while (1) { # sort out times - $begin = strftime('%Y-%m-%d 07:00',localtime); + $begin = strftime('%Y-%m-%d 06:35',localtime); $end = $czarDb->getNowTimestamp(); @@ -112,9 +116,14 @@ $begin = $czarDb->subtractInterval($begin, "1 DAY"); } + + # check nightly science status + print "* Checking nightly science status\n"; + if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";} + $czarDb->updateNightlyScience($nsStatus); # check nebulous print "* Checking Nebulous\n"; $nebulous->updateClusterSpaceInfo(); - $czarplot->plotDiskUsageHistogram(); + $plotter->plotDiskUsageHistogram(); updateServerStatus(); @@ -177,5 +186,5 @@ chomp($label); - $czarplot->createTimeSeries($label, $stage, $begin, $end); + $plotter->createLogAndLinearTimeSeries($label, $stage, $begin, $end); } } @@ -185,14 +194,14 @@ my ($label) = @{$row}; - $czarplot->createTimeSeries($label, undef, $begin, $end); - $czarplot->createHistogram($label, $begin, $end); + $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end); + $plotter->createHistogram($label, $begin, $end); #routineChecks($label, "1 HOUR"); } - $czarplot->createTimeSeries("all_".$server."_labels", undef, $begin, $end); - $czarplot->createHistogram("all_".$server."_labels", $begin, $end); + $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end); + $plotter->createHistogram("all_".$server."_labels", $begin, $end); foreach $stage (@stages) { - $czarplot->createTimeSeries("all_".$server."_labels", $stage, $begin, $end); # TODO must be a neater way... + $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", $stage, $begin, $end); # TODO must be a neater way... } } @@ -233,8 +242,19 @@ chomp($label); - $new = $gpc1Db->countExposures($label, $stage, $newState); - $full = $gpc1Db->countExposures($label, $stage, "full"); - $faults = $gpc1Db->countFaults($label, $stage, $newState); - + if ($stage eq "burntool") { + + if ($labelServer eq "stdscience") { + + $burntool->getPendingAndProcessed($label, \$new, \$full); + $faults = 0; + } + else { $new = $full = $faults = 0;} + } + else { + + $new = $gpc1Db->countExposures($label, $stage, $newState); + $full = $gpc1Db->countExposures($label, $stage, "full"); + $faults = $gpc1Db->countFaults($label, $stage, $newState); + } #printf("%s %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults); $totalNew += $new; Index: /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster.sh =================================================================== --- /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster.sh (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster.sh (revision 29124) @@ -0,0 +1,53 @@ +#!/bin/bash -f + +# +# This script is supposed to be run as ipp user (on ipp004) +# +# for each computer of the cluster +# get the result of the command 'top -d 3 -b -n 5 -i' +# i.e.: +# - show active processes (-i) +# - 5 times (-n 5) +# - run in batch mode (-b) +# - every 3 seconds (-d 3) +# +# If nothing is changed in this script, data can be analyzed using +# analyze_logs.pl +# + +# +# TODO: +# - make it parametered? +# - add an option for ssh timeout +# - add it to crontab +# - split hosts by pantasks server? +# + +# The hosts list on which we're sshing +HOSTS="ippdb00 ippdb01 ippdb02\ + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09\ + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19\ + ipp004 ipp005 ipp006 ipp007 ipp008 ipp009\ + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019\ + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029\ + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039\ + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049\ + ipp050 ipp051 ipp052 ipp053" + +# Basename of the files +LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_" + +# Run +for computer in $HOSTS; do + LOGFILE=$LOGFILE_BASE$computer + # Delete existing logfile + echo "" > $LOGFILE + echo "########### $computer ###########" + echo "########### $computer ###########" >> $LOGFILE +# /usr/bin/ssh $computer 'setenv TERM xterm; top -d 3 -b -n 5 -i -c' >> $LOGFILE 2> /dev/null & + /usr/bin/ssh $computer 'setenv TERM xterm; setenv COLUMNS 1000; top -d 3 -b -n 5 -c -i' >> $LOGFILE 2> /dev/null & +done + +# +# We should wait 15 to 20 seconds to get the results. The perl script uses any input file though. +# Index: /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster_analyze.pl =================================================================== --- /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster_analyze.pl (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/_who_uses_the_cluster_analyze.pl (revision 29124) @@ -0,0 +1,138 @@ +#!/usr/local/bin/perl -w + +# +# This script analyzes the results of the who_uses_the_system command +# + +use File::stat; + +# Basename for log files +$LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_"; +# Hosts on which the 'who_uses_the_system' command was run +@HOSTS=qw/ipp004 ipp005 ipp006 ipp007 ipp008 ipp009 + ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019 + ipp020 ipp021 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 + ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039 + ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049 + ipp050 ipp051 ipp052 ipp053 + ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09 + ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19 + ippdb00 ippdb01 ippdb02/; + +print "\n"; +print "Who uses the system?\n"; +print "\n"; +print "To get info, move the mouse pointer over the table cell. !Warning! Command lines can be truncated


\n"; +print "\n"; +# Scan each logfile +foreach $computer (@HOSTS) { + $LOGFILE=$LOGFILE_BASE.$computer; + $canopen = open LOG, "<$LOGFILE"; + if ($canopen) { # The logfile exists (host is active) + # The summary table is indexed by the pid. for each pid, we + # store the average cpu/memory load, the process name, the + # user name + %summary = (); + for $line () { + chomp $line; + if ( !($line =~ /.*top.*/) && ($line =~ /^ ?[1-9].*/) ) { + # We don't want the 'top' process to be shown + $line =~ s/\s+/;/g; + $line =~ s/^;//g; + # print "$line\n"; + @values = split ';', $line; + $pid = $values[0]; + #print $pid, "\n"; + $username = $values[1]; + $proc =$values[8]; + $mem =$values[9]; + $size = scalar(@values); + #$procname = $values[11]; + $fullprocname = ""; + #print "$procname \n"; + for ($i=11;$i<$size;$i++) { + $fullprocname = $fullprocname.$values[$i]." "; + } + if (!defined $summary{$pid}) { + # The pid is a new one + $summary{$pid}{"count"} = 1; + $summary{$pid}{"username"} = $username; + $summary{$pid}{"proc"} = $proc; + $summary{$pid}{"memory"} = $mem; + $summary{$pid}{"procname"} = $fullprocname; + } else { + # The pid has already been observed + # print "The pid has already been observed\n"; + #if ($summary{$pid}{"procname"} eq $procname) { + $before = $summary{$pid}{"count"}; + $summary{$pid}{"count"}++; + $after = $summary{$pid}{"count"}; + $ratio = $before / $after; + $summary{$pid}{"proc"} = $ratio * + $summary{$pid}{"proc"} + $proc / $after; + $summary{$pid}{"memory"} = $ratio * + $summary{$pid}{"memory"} + $proc / $after; + #} + } + } + } + close LOG; + # When was the logfile produced? mtime is modification time + $stats = stat($LOGFILE); + # Show results + printf("\n", + $LOGFILE, scalar localtime $stats->mtime, $computer); + $onceShown = 0; + while ( my ($pid, $value) = each(%summary) ) { + # print $value->{"count"} , " ", $pid, "\n"; + if ($value->{"count"} > 1) { + #print "Ok\n"; + $onceShown = 1; + printf("\n"); + if ($value->{"username"} eq "ipp" or $value->{"username"} eq "apache" or $value->{"username"} eq "root") { + $color = "black"; + } else { + # The user us not a regular one... Show it in red. + $color = "red"; + } + $comment = "Whom the process belongs to"; + printf("\n", $value->{"username"}); + $comment = "Process name"; + printf("\n", $value->{"procname"}); + if ($value->{"proc"} > 100.) { + $color = "red"; + } elsif ($value->{"proc"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average cpu load"; + printf("\n", $value->{"proc"}); + if ($value->{"memory"} > 100.) { + $color = "red"; + } elsif ($value->{"memory"} > 50.) { + $color = "orange"; + } else { + $color = "black"; + } + $comment = "Average memory load"; + printf("\n", $value->{"memory"}); + $comment = "How many times the process was observed every 3 seconds during 15 seconds"; + printf("\n", $value->{"count"}); + $comment = "Process pid"; + printf("\n", $pid); + printf("\n"); + } + } + if ($onceShown == 0) { + print "\n"; + } + } else { + print "\n"; + } +} + +print "
%s
%s%s%.1f%.1f%s%s
Nothing interesting for this one
$computerCan't open $LOGFILE
\n"; +print "\n"; +print "\n"; + Index: /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh =================================================================== --- /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh (revision 29124) +++ /branches/eam_branches/ipp-20100823/tools/who_uses_the_cluster/who_uses_the_cluster_analyze.sh (revision 29124) @@ -0,0 +1,10 @@ +#!/bin/bash + +INSTALLDIR=/home/panstarrs/ipp/who_uses_the_cluster +$INSTALLDIR/_who_uses_the_cluster.sh +echo "Sleeping 17s" +sleep 17 +CLUSTERMONITORDIR=~/htdocs/clusterMonitor +mkdir -p $CLUSTERMONITORDIR +/bin/rm -f $CLUSTERMONITORDIR/top.html +$INSTALLDIR/_who_uses_the_cluster_analyze.pl > $CLUSTERMONITORDIR/top.html