Index: /branches/eam_branches/20091201/tools/dvo2mass.sh =================================================================== --- /branches/eam_branches/20091201/tools/dvo2mass.sh (revision 26852) +++ /branches/eam_branches/20091201/tools/dvo2mass.sh (revision 26852) @@ -0,0 +1,64 @@ +#!/usr/bin/env dvo + +macro go + if ($0 != 5) + echo "USAGE: go (RA) (DEC) (RADIUS) (output)" + break + end + + # sleep 10 + catdir /data/ipp004.0/ipp/ippRefs/catdir.synth.grizyJHK + + $rmin = $1 - $3*dcos($2) + $rmax = $1 + $3*dcos($2) + $dmin = $2 - $3 + $dmax = $2 + $3 + if ($3 + $2 >= 90) + $rmin = 0.0 + $rmax = 360.0 + $dmax = 90.0 + end + if ($3 + $2 <= -90.0) + $rmin = 0.0 + $rmax = 360.0 + $dmin = -90.0 + end + + if ($rmin < 0.0) + skyregion 0.0 $rmax $dmin $dmax + avextract ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + write $4 ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + + skyregion {$rmin + 360.0} 360.0 $dmin $dmax + avextract ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + write -append $4 ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + return + end + + if ($rmax > 360.0) + echo skyregion 0.0 {$rmax - 360.0} $dmin $dmax + skyregion 0.0 {$rmax - 360.0} $dmin $dmax + avextract ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + write $4 ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + + echo skyregion $rmin 360.0 $dmin $dmax + skyregion $rmin 360.0 $dmin $dmax + avextract ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + write -append $4 ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + return + end + + skyregion $rmin $rmax $dmin $dmax + avextract ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K + write $4 ra dec g_SYNTH r_SYNTH i_SYNTH z_SYNTH y_SYNTH J H K +end + +if ($SCRIPT) + if ($argv:n != 4) + echo "USAGE: dvo2mass (RA) (DEC) (RADIUS) (output)" + exit 2 + end + + go $argv:0 $argv:1 $argv:2 $argv:3 + exit 0 +end Index: /branches/eam_branches/20091201/tools/examine_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/20091201/tools/examine_burntool_pcontrol.pl (revision 26851) +++ /branches/eam_branches/20091201/tools/examine_burntool_pcontrol.pl (revision 26852) @@ -17,7 +17,8 @@ print STDERR " status of the processing. On the OTA grid:\n"; print STDERR " _ Blank chip\n"; - print STDERR " N Null/zero value for burntool_state. No burntool attempted.\n"; - print STDERR " O -1 value for burntool_state. Burntool is running.\n"; - print STDERR " E -2 value for burntool_state. Error!\n"; + print STDERR " O Null/zero value for burntool_state. No burntool attempted (yet).\n"; + print STDERR " P -1 value for burntool_state. Burntool is running.\n"; + print STDERR " E -2 value for burntool_state. Error! (modified raw pixels)\n"; + print STDERR " X -3 value for burntool_state. Burntool segfaulted on this chip.\n"; print STDERR " B Current version value for burntool_state. Burntool succeeded.\n"; print STDERR " b Old version value for burntool_state. Burntool needs to be rerun.\n"; @@ -51,17 +52,17 @@ $burncount++; } - elsif ($burntool_state == -1) { + elsif ($burntool_state == -1) { # processing $char = 'P'; } - elsif ($burntool_state == 0) { + elsif ($burntool_state == 0) { # has not been processed yet $char = 'O'; } - elsif ($burntool_state == -2) { + elsif ($burntool_state == -2) { # modified pixel data $char = 'E'; } - elsif ($burntool_state == -3) { + elsif ($burntool_state == -3) { # previous burntool segfaulted. $char = 'X'; } - else { + else { # old version $char = 'b'; } @@ -128,5 +129,4 @@ $data_ref = $db->selectall_arrayref( $sth ); - # Reset Index: /branches/eam_branches/20091201/tools/gpc1_find_images.pl =================================================================== --- /branches/eam_branches/20091201/tools/gpc1_find_images.pl (revision 26851) +++ /branches/eam_branches/20091201/tools/gpc1_find_images.pl (revision 26852) @@ -3,5 +3,5 @@ use DBI; use Getopt::Std; -getopts('hFARCPp:BLMDWIvx:n:c:w:s:O:S:N:',\%opt); +getopts('hEFARCPp:BLMDWIJVYTvx:n:c:w:s:d:e:O:S:N:',\%opt); #getopt('xncwOS:hARCW',\%opt); unless (exists($opt{O})) { @@ -34,10 +34,17 @@ } +# abcdefghijklmnopqrstuvwxyz +#used: +# ABCDEF LMNOP RST W Y +# c h n s vwx if (exists($opt{h})) { print "USAGE find_images.pl ... \n"; print " -h This help\n"; - print " -R Return raw data.\n"; - print " -C Return chip stage data.\n"; - print " -W Return warp stage data.\n"; + print " -R Return Raw data.\n"; + print " -C Return Chip stage data.\n"; + print " -E Return camEra stage data.\n"; + print " -W Return Warp stage data.\n"; + print " -T Get the sTacks for a given input.\n"; + print " -J Get the dIff stage data (warps only for now).\n"; print " -A Return not just fits images.\n"; print " -F Convert to real filenames.\n"; @@ -45,11 +52,16 @@ print " -B Return burntool table.\n"; print " -M Include masks in results.\n"; + print " -L Return the log files from this stage.\n"; print " -D Return only pre-destreaked images.\n"; + print " -Y Yes, I want the raw, even if it's not been processed.\n"; + print " -V Return the cmf or smf files.\n"; print " -v Be verbose on output.\n"; print " \n"; print " -x Specify a exp_id.\n"; print " -c ...\n"; + print " -e ...\n"; print " -w ...\n"; print " -s ...\n"; + print " -d ...\n"; print " -n ...\n"; print " \n"; @@ -64,6 +76,6 @@ $dbname = 'gpc1'; $dbserver = 'ippdb01'; -$dbuser = 'ipp'; -$dbpass = 'ipp'; +$dbuser = 'ippuser'; +$dbpass = 'ippuser'; $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . "mysql_socket=" . DB_SOCKET(), @@ -72,5 +84,10 @@ ) or die "Unable to connect to database $DBI::errstr\n"; -$sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack' from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)"; +if ($opt{Y}) { + $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun RIGHT OUTER JOIN fakeRun USING(fake_id) RIGHT OUTER JOIN camRun USING(cam_id) RIGHT OUTER JOIN chipRun USING(chip_id) RIGHT OUTER JOIN rawExp USING (exp_id)"; +} +else { + $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)"; +} # What do we have? @@ -89,10 +106,28 @@ push @sth_adds, "warpRun.warp_id = $opt{w}"; } -if (exists($opt{s})) { - if ($opt{s} != 0) { +if (exists($opt{e})) { + push @sth_adds, "camRun.cam_id = $opt{e}"; +} +if (exists($opt{s})||exists($opt{T})) { +# print "$opt{s}!\n"; + if (exists($opt{s})&&($opt{s} != 0)) { push @sth_adds, "stackRun.stack_id = $opt{s}"; } - $sth_base =~ s/0,"nostack"/stack_id,stackSumSkyfile.uri/; + $sth_base =~ s/0,'nostack'/stack_id,stackSumSkyfile.uri/; + if (exists($opt{T})) { + $sth_base .= ' RIGHT OUTER '; + } $sth_base .= ' JOIN stackInputSkyfile USING (warp_id) JOIN stackSumSkyfile USING (stack_id) JOIN stackRun USING (stack_id)'; +# print "$sth_base\n"; +} +if (exists($opt{J})||(exists($opt{d}))) { + if (exists($opt{d})&&($opt{d} != 0)) { + push @sth_adds, "diffRun.diff_id = $opt{d}"; + } + $sth_base =~ s/0,'nodiff'/diffRun.diff_id,diffSkyfile.path_base/; + if (exists($opt{J})) { +# $sth_base .= ' RIGHT OUTER '; + } + $sth_base .= ' JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) JOIN diffRun USING (diff_id) JOIN diffSkyfile USING (diff_id)'; } if ($#sth_adds != -1) { @@ -112,12 +147,17 @@ ($warp_id, $warp_workdir, $chip_id, $chip_workdir, $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime, - $stack_id, $stack_sumfile_uri) = @{ $rr }; + $stack_id, $stack_sumfile_uri, + $cam_id, $cam_workdir, + $diff_id, $diff_workdir + ) = @{ $rr }; if ($stack_id == 0) { $stack_sumfile_uri = ''; } -# print (join "\t", @{ $rr }) . "\n"; +# print (join "\t", @{ $rr }) . "\n" . "\n"; # $exp_pontime = 0; # To stop warnings, but I want this around. if (exists($opt{I})) { $stack_id = 0; # Ditto. + $cam_id = 0; + $diff_id = 0; } if (exists($opt{N})) { @@ -136,4 +176,5 @@ if (exists($opt{B})) { $previous_cmd .= ' -B '; } if (exists($opt{v})) { $previous_cmd .= ' -v '; } + if (exists($opt{F})) { $previous_cmd .= ' -F '; } if ($opt{A}) { $previous_cmd .= ' -A '; } if ($opt{M}) { $previous_cmd .= ' -M '; } @@ -156,4 +197,5 @@ (exists($opt{W})&&(exists($opt{O})))|| exists($opt{I})) { + $sth_mapping = "SELECT skycell_id,class_id from warpSkyCellMap WHERE warp_id = $warp_id"; $mr = $db->selectall_arrayref( $sth_mapping ); @@ -166,4 +208,5 @@ push @{ $mapping{$class} }, $skycell; push @{ $mapping{$raw_class} }, $skycell; +# print STDERR "$raw_class $class $skycell\n"; } } @@ -217,4 +260,15 @@ close(NLS); } + if (exists($opt{E})) { + open(NLS,"neb-ls $cam_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; + while () { + chomp; + if (is_valid_cam($_)) { + push @scan, $_; + $found++; + } + } + close(NLS); + } if (exists($opt{W})) { open(NLS,"neb-ls $warp_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; @@ -227,7 +281,23 @@ } } - if (exists($opt{s})) { - push @scan, $stack_sumfile_uri; - $found++; + if (exists($opt{J})) { + @diff_tmp = split /\//, $diff_workdir; + $diff_prefix = pop(@diff_tmp); + $diff_true_dir = join '/',@diff_tmp; + + open(NLS,"neb-ls $diff_true_dir |") || die "can't neb-ls"; + while () { + chomp; + if (is_valid_diff($_)) { + push @scan, $_; + $found++; + } + } + } + if (exists($opt{s})||exists($opt{T})) { + if (defined($stack_sumfile_uri)) { + push @scan, $stack_sumfile_uri; + $found++; + } } @@ -277,4 +347,10 @@ elsif ($opt{B}) { if ($key =~ /burn.tbl/) { + if (($opt{O} ne '')&&($key !~ /$opt{O}/)) { + return(0); # Not the requested OTA + } + if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) { + return(0); # Not the remapped skycell requested + } return(1); } @@ -332,4 +408,7 @@ } } + elsif ($opt{V} && ($key =~ /cmf/)) { + return(1); + } elsif ($key !~ /fits/) { return(0); # Not a fits @@ -353,4 +432,68 @@ } +sub is_valid_cam { + my $key = shift; + if ($key =~ /\.cm\./) { + if ($opt{A}) { + return(1); + } + elsif ($opt{L} && ($key =~ /log/)) { + return(1); + } + elsif ($opt{V} && ($key =~ /smf/)) { + return(1); + } + } + return(0); +} + +sub is_valid_diff { + my $key = shift; + if ($key =~ /$diff_prefix/) { + if ($opt{D} && ($key !~ /SR/)) { + return(0); # Not destreaked when required + } + if (($opt{S} ne '')&&($key !~ /$opt{S}/)) { + return(0); # Not the requested skycell + } + if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) { + return(0); # Not the remapped OTA requested + } + + if ($opt{A}) { + return(1); # You asked for all, + } + elsif ($opt{L}) { + if ($key =~ /log/) { + return(1); + } + else { + return(0); + } + } + elsif ($opt{V} && ($key =~ /cmf/)) { + return(1); + } + elsif ($key !~ /fits/) { + return(0); # Not a fits + } + elsif ($key =~ /wt.fits/) { + return(0); # No handler for weight right now. + } + elsif ($key =~ /mask/) { + if ($opt{M}) { + return(1); # Requested mask + } + else { + return(0); + } + } + else { + return(1); # Should be just the warp. + } + } + + return(0); +} sub is_valid_warp { my $key = shift; @@ -380,4 +523,7 @@ } } + elsif ($opt{V} && ($key =~ /cmf/)) { + return(1); + } elsif ($key !~ /fits/) { return(0); # Not a fits @@ -399,7 +545,2 @@ return(0); } - - - - - Index: /branches/eam_branches/20091201/tools/make_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/20091201/tools/make_burntool_pcontrol.pl (revision 26851) +++ /branches/eam_branches/20091201/tools/make_burntool_pcontrol.pl (revision 26852) @@ -35,4 +35,20 @@ ) or die "Unable to connect to database $DBI::errstr\n"; +# Determine what the value of "BURNTOOL.STATE.GOOD" currently is: +$burntoolStateGood = 999; +open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; +while() { + chomp; + if ($_ =~ /BURNTOOL.STATE.GOOD/) { + @line = split /\s+/; + $burntoolStateGood = $line[2]; + } +} +close(LAZY); +#unless( $burntoolStateGood == 999) { +# print STDERR "GOOD == $burntoolStateGood\n"; +#} + + # List of which obs_modes are "science" and which are something else. %science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, @@ -41,8 +57,11 @@ $science{Unknown} = 1; } -# Zero the arrays. + +# Zero the arrays and counters @dates_min = (); @dates_max = (); - +$N_burntooled_images = 0; +$N_total_images = 0; +$N_ranges = 0; # Parse SQL query and fill any valid dates to the arrays if (exists($opt{Q})) { @@ -66,4 +85,5 @@ } } + # Read a single date, and calculate the end and insert elsif (exists($opt{d})) { @@ -73,4 +93,5 @@ push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); } + # Just shift the supplied values into the arrays. else { @@ -92,5 +113,5 @@ print "##query: SELECT dateobs FROM rawExp WHERE $opt{Q}\n"; } -#for ($d = 0; $d <= $#dates_min; $d++) { + while ($#dates_min > -1) { $date_min = shift(@dates_min); @@ -138,14 +159,10 @@ $science{$obs_mode} = 0; } -#%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, -# 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 0, ' ' => 0); -# I don't like this bit, but we need to work around the bad values of obs_mode that periodically crop up. -# This isn't the most robust solution (which would of course be "having trustworthy obs_mode values"), but -# it'll probably work for now. if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| (($science{$obs_mode} == 0)&&( ($comment =~ /MD/)||($comment =~ /ThreePi/)|| ($comment =~ /STS/)||($comment =~ /M31/)|| - ($comment =~ /CAL/)||($comment =~ /SS/)||($comment =~ /SVS/)|| ($comment =~ /Sweetspot/)|| ($comment =~ /cal/)|| ($comment =~ /virgo/)|| ($comment =~ /Virgo/)|| ($comment =~ /kepler/)|| + ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)|| + ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)|| ($comment =~ /sdss/)))) { @@ -174,4 +191,5 @@ next; } + # Scan again and count how many exposures are between windows foreach $row_ref (@{ $data_ref }) { @@ -212,7 +230,5 @@ } } -# print ">>>$i $start_s[$i] $skips[$i]\n"; - } -# print ">>> $dateobs\n"; + } } @@ -227,17 +243,38 @@ } # Print out the end points in the format burntool wants. + for ($i = 0; $i <= $#start_s; $i++) { @start_t = localtime($start_s[$i]); @end_t = localtime($end_s[$i]); - printf(" burntool %04d-%02d-%02dT%02d:%02d:%02d %04d-%02d-%02dT%02d:%02d:%02d\n", - $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], - $start_t[2],$start_t[1],$start_t[0], - $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], - $end_t[2],$end_t[1],$end_t[0]); - } - print "\n"; -} - -unless(exists($opt{b})) { + + my $date_min = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", + $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], + $start_t[2],$start_t[1],$start_t[0]); + my $date_max = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", + $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], + $end_t[2],$end_t[1],$end_t[0]); + + my $total_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max'"; + my $im_ref = $db->selectall_arrayref( $total_images_sth ); + $N_total_images += ${ $im_ref }[0][0]; + + my $burntooled_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max' AND abs(burntool_state) = $burntoolStateGood"; + $im_ref = $db->selectall_arrayref( $burntooled_images_sth ); + $N_burntooled_images += ${ $im_ref }[0][0]; + + + printf(" burntool %s %s\n",$date_min,$date_max); + } + unless(exists($opt{d})) { + print "\n"; + } + $N_ranges += ($#start_s + 1); +} + +if ($N_ranges == 0) { + print_short_epilogue(); +} + +unless(exists($opt{b}) || ($N_ranges == 0)) { print_epilogue(); } @@ -275,6 +312,20 @@ return(0); } +sub print_short_epilogue { + if (exists($opt{b})) { + print STDERR "There were no science exposures to process. Sorry, try again tomorrow.\n"; + return(); + } + print << 'END_SHORT_EPILOGUE'; + + echo "There were no science exposures to process. Sorry, try again tomorrow." + +end +END_SHORT_EPILOGUE +} sub print_epilogue { + print " echo 'There were $N_total_images total images, of which $N_burntooled_images were already burntooled.'\n"; + print "### BTSTAT $N_total_images $N_burntooled_images\n"; print << 'END_EPILOGUE'; Index: /branches/eam_branches/20091201/tools/neb-ds9.pl =================================================================== --- /branches/eam_branches/20091201/tools/neb-ds9.pl (revision 26851) +++ /branches/eam_branches/20091201/tools/neb-ds9.pl (revision 26852) @@ -4,5 +4,5 @@ getopts('AM',\%opt); -$cmd = 'ds9 -scale mode zscale '; +$cmd = 'ds9 -scale mode zscale '; if (exists($opt{A})) { @@ -21,3 +21,4 @@ } } +#$cmd .= " -zoom to fit -regions format xy -regions system wcs -regions load targets.reg "; system("$cmd"); Index: /branches/eam_branches/20091201/tools/warp_outputs.pl =================================================================== --- /branches/eam_branches/20091201/tools/warp_outputs.pl (revision 26851) +++ /branches/eam_branches/20091201/tools/warp_outputs.pl (revision 26852) @@ -24,4 +24,5 @@ my ($input); # Input list my ($products); # Products of interest +my ($muggle); # No magicked inputs? GetOptions( @@ -33,5 +34,7 @@ 'skycell_id=s' => \$skycell_id, # Skycell identifier 'products=s' => \$products, # Products of interest - ) or die "Unable to parse arguments.\n"; + 'muggle' => \$muggle, # No magicked inputs? + ) or +die "Unable to parse arguments.\n"; die "Unknown option: @ARGV\n" if @ARGV; die "Required options: --dbhost --dbname --dbuser --dbpass --warp_id\n" @@ -80,4 +83,12 @@ my $file = "$path$ext"; # File + + if (defined $muggle and $ext ne ".psf") { + my @file = split /\//, $file; + my $last = pop @file; + push @file, "SR_$last"; + $file = join '/', @file; + } + my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file chomp $source; Index: /branches/eam_branches/20091201/tools/wiki_nightly_stacks_table.pl =================================================================== --- /branches/eam_branches/20091201/tools/wiki_nightly_stacks_table.pl (revision 26852) +++ /branches/eam_branches/20091201/tools/wiki_nightly_stacks_table.pl (revision 26852) @@ -0,0 +1,326 @@ +#! /usr/local/bin/perl -w + +use DBI; +use Getopt::Std; + +getopts('PC',\%opt); + +$dbname = 'gpc1'; +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} + +$db = init_gpc_db(); + +@targets = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10'); + +# potential targets +if (exists($opt{P})) { + foreach $t (@targets) { + $sth = "select filter,DATE_FORMAT(rawExp.dateobs,'%Y-%m-%d') AS day,count(filter) FROM rawExp JOIN newExp using(exp_id) WHERE rawExp.fault = 0 AND exp_type = 'OBJECT' AND comment LIKE '${t}%' AND dateobs >= '2009-12-08T00:00:00' GROUP BY day,filter"; + $data_ref = $db->selectall_arrayref( $sth ); + foreach $rr (@{ $data_ref }) { + ($filter,$date,$N) = @{ $rr }; + $filter =~ s/.00000//; + $potential{$date}{$t}{$filter} = $N; + } + } + print "|| '''Potential (Nexp):'''\n||\n"; + print '|| Date ||'; + foreach $t (@targets) { + print "$t ||"; + } + print "\n"; + + foreach $d (sort (keys %potential)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $potential{$d}{$t} })) { + if ($i == 0) { + print "${f}($potential{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($potential{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; + } + print "||\n"; +} +#queued in chip + +$sth = "select rawExp.dateobs,chipRun.state,rawExp.filter,count(chipRun.state),chipRun.data_group,chipRun.dist_group from chipRun JOIN rawExp using (exp_id) where chipRun.label LIKE '%nightlystack' GROUP BY data_group,filter,state ORDER BY dateobs"; + +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($date,$state,$filter,$N,$data_group,$dist_group) = @{ $rr }; + if ($state eq 'full') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $chips{$date}{$dist_group}{$filter} = $N; + } + if ($state eq 'new') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $queued{$date}{$dist_group}{$filter} = $N; + } + +} +print "|| '''Queued for Chip (Nexp):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %queued)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $queued{$d}{$t} })) { + if ($i == 0) { + print "${f}($queued{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($queued{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; +print "|| '''Through Chip (Nexp):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %chips)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $chips{$d}{$t} })) { + if ($i == 0) { + print "${f}($chips{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($chips{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +#warps +$sth = "select rawExp.dateobs,warpRun.state,rawExp.filter,count(warpRun.state),warpRun.data_group,warpRun.dist_group from warpRun JOIN fakeRun using (fake_id) JOIN camRun using (cam_id) JOIN chipRun using (chip_id) JOIN rawExp using (exp_id) where warpRun.label LIKE '%nightlystack' GROUP BY data_group,filter,state ORDER BY data_group,filter"; +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($date,$state,$filter,$N,$data_group,$dist_group) = @{ $rr }; + if ($state eq 'full') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $data{$date}{$dist_group}{$filter} = $N; + } +} + +print "|| '''Through Warp (Nexp):'''\n||\n"; + +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %data)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $data{$d}{$t} })) { + if ($i == 0) { + print "${f}($data{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($data{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +# stacks + + +$sth = "select state,count(skycell_id),data_group,dist_group,filter from stackRun where label LIKE '%nightlystack' group by data_group,filter,state"; +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($state,$N,$data_group,$dist_group,$filter) = @{ $rr }; + $filter =~ s/.00000//; + if ($state eq 'full') { + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + $stacks{$date}{$dist_group}{$filter} = $N; + } + if ($state eq 'new') { + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + $queuestacks{$date}{$dist_group}{$filter} = $N; + } +} +print "|| '''Queued Stack (Nskycells):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %queuestacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $queuestacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($queuestacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($queuestacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; +print "|| '''Through Stack (Nskycells):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %stacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $stacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($stacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($stacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +# distribution + +$sth = "select distRun.data_group,dist_group,filter,count(stage_id) from distRun JOIN stackRun ON (stage_id = stack_id) where distRun.label LIKE '%.nightlystack' AND distRun.state = 'full' GROUP BY distRun.data_group,filter"; + +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($data_group,$dist_group,$filter,$N) = @{ $rr }; + $filter =~ s/.00000//; + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + + $diststacks{$date}{$dist_group}{$filter} = $N; +} +print "|| '''Distributed (Nstack_ids):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %diststacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $diststacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($diststacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($diststacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print '||' . "\n"; +# faulted distributed + +$sth = "select * from rcDSFileset join distRun using(dist_id) where rcDSFileset.fault > 0 and date(registered) >= '2010-01-01' AND label LIKE 'M%.nightlystack' order by fs_id asc"; +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { +# ($dist_id,$fs_id,$dest_id,$name,$state,$registered,$fault,$target_id,$stage,$stage_id,$magic_ds_id,$label,$outroot,$outdir,$clean,$no_magic,$state2,$time_stamp,$fault,$data_group,$note) = @{ $rr }; + + $data_group = ${ $rr }[19]; + ($dist_group,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + + unless(defined($fault_diststacks{$date}{$dist_group})) { + $fault_diststacks{$date}{$dist_group} = 0; + } + + $fault_diststacks{$date}{$dist_group} ++; + + if (exists($opt{C})) { + print STDERR "## disttool -dbname gpc1 -revertfileset -fs_id ${ $rr }[1]\n"; + } + +} + +print "|| '''Faulted Dists (Nstack_ids):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %diststacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + if (exists($fault_diststacks{$d}{$t})) { + print "$fault_diststacks{$d}{$t}"; + } + } + print '||' . "\n"; +} +print "\n\n";