Index: /branches/eam_branches/20090715/tools/diff_outputs.pl =================================================================== --- /branches/eam_branches/20090715/tools/diff_outputs.pl (revision 25397) +++ /branches/eam_branches/20090715/tools/diff_outputs.pl (revision 25398) @@ -16,9 +16,9 @@ 'SOURCES', 'JPEG1', 'JPEG2', 'KERNEL' ], - 'ALL' => [ 'IMAGE', 'MASK', 'VARIANCE', - 'SOURCES', 'JPEG1', 'JPEG2', - 'KERNEL', 'INVERSE.IMAGE', - 'INVERSE.MASK', 'INVERSE.VARIANCE', - 'INVERSE.SOURCES' ], + 'ALL' => [ 'IMAGE', 'MASK', 'VARIANCE', 'SOURCES', 'JPEG1', 'JPEG2', 'KERNEL', + 'INVERSE.IMAGE', 'INVERSE.MASK', 'INVERSE.VARIANCE', 'INVERSE.SOURCES', + 'INCONV.IMAGE', 'INCONV.MASK', 'INCONV.VARIANCE', + 'REFCONV.IMAGE', 'REFCONV.MASK', 'REFCONV.VARIANCE', + ], 'INVERSE' => [ 'INVERSE.IMAGE', 'INVERSE.MASK', 'INVERSE.VARIANCE', @@ -38,4 +38,10 @@ 'INVERSE.VARIANCE' => '.inv.wt.fits', # Variance 'INVERSE.SOURCES' => '.inv.cmf', # Sources + 'INCONV.IMAGE' => '.inConv.fits', # Convolved input image + 'INCONV.MASK' => '.inConv.mk.fits', # Convolved input mask + 'INCONV.VARIANCE' => '.inConv.wt.fits', # Convolved input variance + 'REFCONV.IMAGE' => '.refConv.fits', # Convolved reference image + 'REFCONV.MASK' => '.refConv.mk.fits', # Convolved reference mask + 'REFCONV.VARIANCE' => '.refConv.wt.fits', # Convolved reference variance }; Index: /branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl (revision 25398) +++ /branches/eam_branches/20090715/tools/examine_burntool_pcontrol.pl (revision 25398) @@ -0,0 +1,181 @@ +#!/usr/bin/perl -w + +# script to print out information about the current status of a burntool run. + +use DBI; +use Getopt::Std; + +getopts('hf',\%opt); +if (exists($opt{h})) { + print STDERR "Usage: examine_burntool_pcontrol.pl [-f] \n"; + print STDERR " -f Read all burntool entries, even those commented out.\n"; + print STDERR "\n"; + print STDERR " Reads in the pcontrol.pro file you're using to run burntool, and uses the\n"; + print STDERR " date ranges included there to query the database to tell you the current\n"; + print STDERR " status of the processing. On the OTA grid:\n"; + print STDERR " _ Blank chip\n"; + print STDERR " N Null value for user_1. No burntool attempted.\n"; + print STDERR " O 0.1 value for user_1. Burntool failed or is running.\n"; + print STDERR " B 1.0 value for user_1. Burntool succeeded.\n"; + exit(1); +} + +# Read a class_id and user_1 pair, and set the correct cell of the (sorry, global) character array vector +sub class_to_vector { + my $class = shift; + my $user1 = shift; + my $char = '_'; + if ($user1 == 1.0) { + $char = 'B'; + $burncount++; + } + elsif ($user1 == 0.1) { + $char = 'O'; + } + elsif ($user1 == -1.0) { + $char = 'N'; + } + + my @cv = split //, $class; + + my $id = $cv[2] * 8 + $cv[3]; + if (($id == 0)||($id == 7)||($id == 63)||($id == 56)) { + die "impossible chip"; + } + $vector[$id] = $char; +} +# Engauge autoflush, in case the database server is acting up. +$| = 1; + + +# Load the pcontrol.pro file, and create arrays of the dates included +$file = shift(@ARGV); +if (!defined($file)) { + system("$0 -h"); + exit(1); +} +else { + @date_min = (); + @date_max = (); + open(F,$file); + while() { + chomp; + if (exists($opt{f})) { + $_ =~ s/^\s*?#//; + } + $_ =~ s/^\s+//; + + if ($_ =~ /^burntool /) { + ($burncount,$start,$end) = split /\s+/; #lazy variable reuse + push @date_min, $start; + push @date_max, $end; + + } + } + close(F); +} + +# Set up the database +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ippdb01'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + + +# Get the data for each date range we're running burntool over +for ($i = 0; $i <= $#date_min; $i++) { + $dmin = $date_min[$i]; + $dmax = $date_max[$i]; + # Get the data. The limit is there to keep the database happy. I don't think you can observe that many + # images on a single night. + $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,user_1,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 60000"; + $data_ref = $db->selectall_arrayref( $sth ); + + + # Reset + $cur_exp_id = -99; + $burncount = 0; + for ($j = 0; $j < 64; $j++) { + $vector[$j] = '_'; + } + + # Print out query for clarity, and the header: + print "#### $sth\n"; + print "# 0 1 2 3 4 5 6 7 \n"; + print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n"; + + # If the database returns no entries (because we have a mistake in pcontrol.pro perhaps), return a null entry. + if ($#{ $data_ref } == -1) { + $cur_exp_id = -99; + $burncount = 0; + for ($j = 0; $j < 64; $j++) { + $vector[$j] = '_'; + } + } + + foreach $row_ref (@{ $data_ref }) { + ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$user_1,$comment) = @{ $row_ref }; + # Correct for nulls in the database. + if (!defined($obs_mode)) { + $obs_mode = 'NULL'; + } + if (!defined($user_1)) { + $user_1 = -1; + } + if (!defined($comment)) { + $comment = ' '; + } + + # If we're still working on the same exp_id, update the chip we got + if ($exp_id == $cur_exp_id) { + class_to_vector($class_id,$user_1); + } + else { + # otherwise, print, reset, and update + $V = join '', @vector; + + if ($cur_exp_id != -99) { + printf("%6d %11s %11s %19s %2d %64s %s\n", + $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment); + $burncount = 0; + for ($j = 0; $j < 64; $j++) { + if (($j == 0)||($j == 7)||($j == 63)||($j == 56)) { + $vector[$j] = '_'; + } + else { + $vector[$j] = '?'; + } + } + } + ($cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$cur_comment) = ($exp_id,$exp_name,$obs_mode,$dateobs,$comment); + class_to_vector($class_id,$user_1); + } + } + # Final entry needs a manual print to clear it out. Let's tack on a footer as well. + printf("%6d %11s %11s %19s %2d %64s %s\n", + $cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$burncount, $V,$cur_comment); + print "# 0 1 2 3 4 5 6 7 \n"; + print "#exp_i exp_name obs_mode dateobs nB 0123456701234567012345670123456701234567012345670123456701234567 comment\n"; + print"\n"; + + # If we didn't find that all 60 chips had succesful burntools run, print out the ipp_apply_burntool.pl command we would need to fix this: + if ($burncount != 60) { + for ($j = 0; $j < 64; $j++) { + $vector[$j] =~ s/\[..m(.).*/$1/; + if ($vector[$j] ne 'B') { + if ($vector[$j] ne '_') { + $id = sprintf("XY%d%d",int($j / 8),$j % 8); + print "#### ipp_apply_burntool.pl --class_id $id --dateobs_begin $dmin --dateobs_end $dmax --dbname gpc1 --logfile ${id}.${dmin}.log\n"; + } + } + } + } + print "\n\n"; +} Index: /branches/eam_branches/20090715/tools/ipp_apply_burntool.pl =================================================================== --- /branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25397) +++ /branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25398) @@ -140,5 +140,6 @@ my $tempfile = new File::Temp ( TEMPLATE => "$basename.XXXX", DIR => '/tmp', - UNLINK => !$save_temps); + UNLINK => !$save_temps, + SUFFIX => '.fits'); my $tmpImfileReal = $tempfile->filename; # print "tmpImfile: $tmpImfile -> $tmpImfileReal\n"; @@ -172,7 +173,7 @@ if ($RAWTABLES) { - $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN); + $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt persist=t", $REALRUN); } else { - $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN); + $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt persist=t", $REALRUN); } if ($status) { Index: /branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl =================================================================== --- /branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl (revision 25398) +++ /branches/eam_branches/20090715/tools/make_burntool_pcontrol.pl (revision 25398) @@ -0,0 +1,395 @@ +#!/usr/bin/perl -w +# script to generate a pcontrol.pro file to run burntool on a set of data. + +use DBI; +use DateTime; +use Getopt::Std; + +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + +getopts('hbQ:d:',\%opt); +if (exists($opt{h})) { + print STDERR "Usage: make_burntool_pcontrol.pl -b {-Q | -d | DATE_MIN0 DATE_MAX0 DATE_MIN1 DATE_MAX1 [...]}\n"; + print STDERR " -Q SQL_WHERE Generate pcontrol from a SQL WHERE query.\n"; + print STDERR " (enclose in single quotes, and use double quotes inside to\n"; + print STDERR " protect from the shell)\n"; + print STDERR " -d DATE Specify a single day to look for.\n"; + print STDERR " -b Only print burntool lines.\n"; + print STDERR "\n"; + print STDERR " Scans the GPC1 database, and identifies \"science\" observations based on the obs_mode.\n"; + print STDERR " Use a 30 minute safety prelude before the first science data to make sure burntool works.\n"; + + exit(1); +} + +# Load database +$dbname = 'gpc1'; +$dbserver = 'ippdb01'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +# List of which obs_modes are "science" and which are something else. +%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, + 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 0, ' ' => 0); + +# Zero the arrays. +@dates_min = (); +@dates_max = (); + +# Parse SQL query and fill any valid dates to the arrays +if (exists($opt{Q})) { + $opt{Q} =~ s/\"/\'/g; + $sth = "SELECT dateobs FROM rawExp WHERE $opt{Q}"; + $data_ref = $db->selectall_arrayref( $sth ); + + %day_keys = (); + foreach $row_ref (@{ $data_ref }) { + ($day,$time) = split /\s+/, ${ $row_ref }[0]; + + unless (exists($day_keys{$day})) { + push @dates_min, $day; + + ($year,$mon,$date) = split /-/, $day; + $date++; + push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); + $day_keys{$day} = 1; +# print ">>>D $day $#dates_min\n"; + } + } +} +# Read a single date, and calculate the end and insert +elsif (exists($opt{d})) { + push @dates_min, $opt{d}; + ($year,$mon,$date) = split /-/, $opt{d}; + $date++; + push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); +} +# Just shift the supplied values into the arrays. +else { + if ($#ARGV == -1) { + system("$0 -h"); + exit(-1); + } + + while ($#ARGV > -1) { + push @dates_min, shift(@ARGV); + push @dates_max, shift(@ARGV); + } +} +unless(exists($opt{b})) { + print_prologue(); +} +# Save the query used in the pcontrol to be safe. +if (exists($opt{Q})) { + print "##query: SELECT dateobs FROM rawExp WHERE $opt{Q}\n"; +} +#for ($d = 0; $d <= $#dates_min; $d++) { +while ($#dates_min > -1) { + $date_min = shift(@dates_min); + $date_max = shift(@dates_max); + + # Find _ALL_ observations within the date range we care about + $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment FROM rawExp WHERE " . + "dateobs >= '${date_min}' AND dateobs <= '${date_max}' order by dateobs"; + + $data_ref = $db->selectall_arrayref( $sth ); + + $seq = 0; + + # Identify the science observations, and make a window earlier for burntool. + @start_s = (0); + @end_s = (0); + @skips = (0); + + foreach $row_ref (@{ $data_ref }) { + + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + # Fix nulls in the database + if (!defined($obs_mode)) { + $obs_mode = ' '; + } + if (!defined($comment)) { + $comment = ' '; + } + + ($date,$time) = split / /, $dateobs; + ($year,$month,$day) = split /-/, $date; + ($hour,$minute,$second) = split /:/, $time; + + $dt = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu'); + + $st = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu')->subtract(minutes => 30); + $et = $dt->epoch(); + + + if (!exists($science{$obs_mode})) { + $science{$obs_mode} = 0; + } + + if (($science{$obs_mode} == 1)&&($comment !~ /Daytime/)) { + if ($start_s[-1] == 0) { + $start_s[-1] = $et - 1800; + $end_s[-1] = $et + 1; + } + else { + if ($et > $end_s[-1] + 1800) { + push @start_s, $et - 1800; + push @end_s, $et + 1; + push @skips, 0; + } + else { + $end_s[-1] = $et + 1; + } + } + } + + $seq++; + + } + + # If this night has no science exposures, skip to the next one. + if (($#start_s == 0) && ($start_s[0] == 0)) { + next; + } + # Scan again and count how many exposures are between windows + foreach $row_ref (@{ $data_ref }) { + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + if (!defined($obs_mode)) { + $obs_mode = ' '; + } + if (!defined($comment)) { + $comment = ' '; + } + + ($date,$time) = split / /, $dateobs; + ($year,$month,$day) = split /-/, $date; + ($hour,$minute,$second) = split /:/, $time; + + $dt = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu'); + + $st = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu')->subtract(minutes => 30); + $et = $dt->epoch(); + + if (!exists($science{$obs_mode})) { + die "No mode >>$obs_mode<<\n"; + } + + for ($i = 0; $i <= $#start_s; $i++) { + if (($et < $start_s[$i])&&($science{$obs_mode} == 0)) { + if ($i == 0) { + $skips[$i]++; + } + else { + if ($et > $end_s[$i-1]) { + $skips[$i]++; + } + } + } +# print ">>>$i $start_s[$i] $skips[$i]\n"; + } +# print ">>> $dateobs\n"; + } + + # Clear out windows that overlap with previous ones so we have the minimum set + for ($i = $#start_s; $i > 0; $i--) { + if ($skips[$i] == 0) { + $trash = pop(@start_s); + $end = pop(@end_s); + $trash = pop(@skips); + $end_s[-1] = $end; + } + } + # Print out the end points in the format burntool wants. + for ($i = 0; $i <= $#start_s; $i++) { + @start_t = localtime($start_s[$i]); + @end_t = localtime($end_s[$i]); + printf(" burntool %04d-%02d-%02dT%02d:%02d:%02d %04d-%02d-%02dT%02d:%02d:%02d\n", + $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], + $start_t[2],$start_t[1],$start_t[0], + $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], + $end_t[2],$end_t[1],$end_t[0]); + } + print "\n"; +} + +unless(exists($opt{b})) { + print_epilogue(); +} + +## +## These functions print out the remaining text of the pcontrol.pro script. I've removed things +## that seemed superfluous. +## + +sub print_prologue { + + print << 'END_PROLOGUE'; +# this script sets up a series of burntool runs targetted to the appropriate machine + +# use the following sql to get the host table match: +# select exp_id, exp_name, class_id, dateobs, user_1, obs_mode, uri from rawImfile where exp_id = 33750 limit 100; + +macro burntool + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + list word -split $hostmatch:$i + $class_id = $word:0 + $logfile = "burntool_logs/$class_id.$1.log" + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile + end +end + +macro go +END_PROLOGUE + +return(0); +} + +sub print_epilogue { + print << 'END_EPILOGUE'; + +end + +macro setnames + $burntool_range:20081001 = 2008-10-01T07:50:00 2008-10-01T15:05:00 +end + +# for a re-run add --skip_burned: +# job -host $word:1 ipp_apply_burntool.pl --class_id $word:0 --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --skip_burned + +macro loadhosts + for i 0 $allhosts:n + host add $allhosts:$i + end +end + +# this macro may be used to complete a burntool run that exited due to error +macro burntool_skip_burned + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + list word -split $hostmatch:$i + $class_id = $word:0 + $logfile = "burntool_logs/$class_id.$1.log" + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --skip_burned --logfile $logfile + end +end + +list hostmatch + XY01 ipp014 + XY02 ipp014 + XY03 ipp038 + XY04 ipp038 + XY05 ipp023 + XY06 ipp023 + XY10 ipp039 + XY11 ipp039 + XY12 ipp024 + XY13 ipp024 + XY14 ipp040 + XY15 ipp040 + XY16 ipp026 + XY17 ipp026 + XY20 ipp041 + XY21 ipp041 + XY22 ipp042 + XY23 ipp042 + XY24 ipp043 + XY25 ipp043 + XY26 ipp028 + XY27 ipp028 + XY30 ipp044 + XY31 ipp044 + XY32 ipp029 + XY33 ipp029 + XY34 ipp045 + XY35 ipp045 + XY36 ipp030 + XY37 ipp030 + XY40 ipp046 + XY41 ipp046 + XY42 ipp031 + XY43 ipp031 + XY44 ipp047 + XY45 ipp047 + XY46 ipp032 + XY47 ipp032 + XY50 ipp048 + XY51 ipp048 + XY52 ipp033 + XY53 ipp033 + XY54 ipp049 + XY55 ipp049 + XY56 ipp034 + XY57 ipp034 + XY60 ipp050 + XY61 ipp050 + XY62 ipp035 + XY63 ipp035 + XY64 ipp051 + XY65 ipp051 + XY66 ipp036 + XY67 ipp036 + XY71 ipp052 + XY72 ipp052 + XY73 ipp015 + XY74 ipp015 + XY75 ipp053 + XY76 ipp053 +end +list allhosts + ipp043 + ipp014 + ipp015 + ipp023 + ipp024 + ipp026 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end + +END_EPILOGUE +return(0); +}