Index: trunk/ippScripts/scripts/chip_imfile.pl =================================================================== --- trunk/ippScripts/scripts/chip_imfile.pl (revision 26831) +++ trunk/ippScripts/scripts/chip_imfile.pl (revision 26901) @@ -134,5 +134,5 @@ $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -146,12 +146,12 @@ if ($useDeburnedImage) { - my $useBestBurntool = metadataLookupBool($recipeData, 'USE.BEST.BURNTOOL'); - - ## Check that we have required programs: -# print STDERR "Inside burntool loop!\n"; - my $regtool = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - + my $useBestBurntool = metadataLookupBool($recipeData, 'USE.BEST.BURNTOOL'); + + ## Check that we have required programs: +# print STDERR "Inside burntool loop!\n"; + my $regtool = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + ## Check the current burntool processing version: my $regtool_state_cmd = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -limit 1"; @@ -173,5 +173,5 @@ my $burntoolState = 999; foreach my $regEntry (@$regData2) { -# print "$regEntry->{exp_id} $regEntry->{burntool_state}\n"; + print "$regEntry->{exp_id} $regEntry->{burntool_state}\n"; if ($regEntry->{exp_id} == $exp_id) { @@ -182,7 +182,7 @@ &my_die("Unable to find burntool_state in metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); } - if ($burntoolState == 32767) { - $burntoolState = 0; - } + if ($burntoolState == 32767) { + $burntoolState = 0; + } ## Read camera config to get the current good burntool state : @@ -190,5 +190,5 @@ my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $camera_config_cmd, verbose => $verbose); + run(command => $camera_config_cmd, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -206,28 +206,29 @@ } - if (abs($burntoolState) != $burntoolStateGood) { - if ($useBestBurntool) { - &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - else { - warn ("Image burntool version does not match current accepted version. Continuing as requested."); - # Internally pretend that we're at a good version. - if ($burntoolState < -10) { - $burntoolState = -1 * $burntoolStateGood; - } - elsif ($burntoolState > 10) { - $burntoolState = $burntoolStateGood; - } - elsif ($burntoolState == 0) { - # You've told me to use a deburned image, and that you don't care if it's the most recent. The database has told me - # that burntool has never been run on this image. We'll hope the database is wrong, but I don't really trust that. - warn ("burntool_state suggests no table will be found. This will likely crash."); - $burntoolState = -1 * $burntoolStateGood; - } - else { - &my_die("No valid burntool table will be found.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - } - } + print "burntool state vs burntoolStateGood : $burntoolState vs $burntoolStateGood\n"; + if (abs($burntoolState) != $burntoolStateGood) { + if ($useBestBurntool) { + &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + else { + warn ("Image burntool version does not match current accepted version. Continuing as requested."); + # Internally pretend that we're at a good version. + if ($burntoolState < -10) { + $burntoolState = -1 * $burntoolStateGood; + } + elsif ($burntoolState > 10) { + $burntoolState = $burntoolStateGood; + } + elsif ($burntoolState == 0) { + # You've told me to use a deburned image, and that you don't care if it's the most recent. The database has told me + # that burntool has never been run on this image. We'll hope the database is wrong, but I don't really trust that. + warn ("burntool_state suggests no table will be found. This will likely crash."); + $burntoolState = -1 * $burntoolStateGood; + } + else { + &my_die("No valid burntool table will be found.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + } ## We now know that we have an image that has been burntooled. @@ -258,8 +259,8 @@ } - $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} apply=t"; + $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t"; } elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table. - $burntool_cmd .= "$tempName apply=t"; + $burntool_cmd .= "$tempName persist=t apply=t"; } else { Index: trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- trunk/ippScripts/scripts/diff_skycell.pl (revision 26831) +++ trunk/ippScripts/scripts/diff_skycell.pl (revision 26901) @@ -55,5 +55,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --diff_id --skycell_id --outroot --run-state", + -msg => "Required options: --diff_id --skycell_id --outroot --run-state --diff_skyfile_id", -exitval => 3, ) unless defined $diff_id @@ -168,4 +168,7 @@ } +print "reduction: $reduction\n"; +print "recipe_ppSub: $recipe_ppSub\n"; + # print "templateMask: $templateMask\n"; # print "templatePath: $templatePath\n"; Index: trunk/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- trunk/ippScripts/scripts/ds9_cmf_regions.pl (revision 26831) +++ trunk/ippScripts/scripts/ds9_cmf_regions.pl (revision 26901) @@ -81,5 +81,5 @@ if ($mag_radius) { foreach my $m (@$mag) { - next unless defined $m and $m < "inf" and $m > "-inf"; + next unless defined $m and $m != "nan" and $m < "inf" and $m > "-inf"; unless (defined $mag_min and defined $mag_max) { $mag_min = $m; @@ -99,5 +99,5 @@ for (my $i = 0; $i < $numRows; $i++) { my $col; # Colour to use - if ($$flags[$i] & $flag) { + if ($$flags[$i] & $flag or not defined $$mag[$i] or $$mag[$i] eq "nan" or $$mag[$i] == "inf" or $$mag[$i] == "-inf") { $numBad++; $col = $flag_colour; @@ -107,5 +107,5 @@ } my $r = $radius; # Radius of circle - next if defined $mag_radius and (not defined $$mag[$i] or $$mag[$i] == "inf" or $$mag[$i] == "-inf"); + next if $mag_radius and (not defined $$mag[$i] or $$mag[$i] eq "nan" or $$mag[$i] == "inf" or $$mag[$i] == "-inf"); $r -= ($$mag[$i] - $mag_min) * $radius_scale if defined $mag_radius; print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$r) \# color = $col\n"; Index: trunk/ippScripts/scripts/ipp_apply_burntool.pl =================================================================== --- trunk/ippScripts/scripts/ipp_apply_burntool.pl (revision 26901) +++ trunk/ippScripts/scripts/ipp_apply_burntool.pl (revision 26901) @@ -0,0 +1,241 @@ +#!/usr/bin/env perl +# this program is used to run 'burntool' on images for a single chip +# for all exposures for a given time period. +# USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD + +# XXX todo: +# - add filters to processedimfile +# + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +#use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $camera, $dbname, $logfile, $verbose, $save_temps); +GetOptions( + 'class_id=s' => \$class_id, # chip identifier + 'dateobs_begin=s' => \$dateobs_begin, # exposure date/time range start + 'dateobs_end=s' => \$dateobs_end, # exposure date/time range stop + 'camera=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --class_id --dateobs_begin --dateobs_end --dbname", + -exitval => 3, + ) unless + defined $class_id and + defined $dateobs_begin and + defined $dateobs_end and + defined $dbname; + +unless (defined $camera) { + $camera = "GPC1"; +# warn("No camera defined. Defaulting to $camera and warning."); +} + +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $funpack = can_run('funpack') or (warn "Can't find funpack" and $missing_tools = 1); +my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1); +#my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# IPP configuration (including nebulous) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up"); + +$ipprc->redirect_output($logfile) if $logfile; + +# Determine the value of a "good" burntool run. +my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); +} + +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); + +my $burntoolStateGood = 999; +foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } +} +if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR); +} +my $outState = -1 * abs($burntoolStateGood); + +print ">>$burntoolStateGood<<\n"; + +# Define list of images to examine. +my $command = "$regtool -processedimfile"; +$command .= " -class_id $class_id"; +$command .= " -dateobs_begin $dateobs_begin"; +$command .= " -dateobs_end $dateobs_end"; +$command .= " -ordered_by_date"; +$command .= " -dbname $dbname" if defined $dbname; + +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +} + +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); + +my $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); + +my $REALRUN = 1; + +# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass +my $processNext = 0; +my $previousTable = ''; + +foreach my $file (@$files) { + my $exp_id = $file->{exp_id}; + my $state = $file->{burntool_state}; + + my $rawImfile = $file->{uri}; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $file->{uri}; + $outTable =~ s/fits$/burn.tbl/; + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $process = 0; + my $previousTableStyle = 0; + + if ($state == 0) { + $process = 1; + } + elsif ($state == -1) { + $process = 1; + } + elsif ($state == -3) { + $process = 1; + } + elsif ($state == -2) { + &my_die("Aborting as $rawImfile has modified pixel data!"); + } + elsif ($state == $burntoolStateGood) { + $process = 0; + $previousTableStyle = 1; + } + elsif ($state == -1 * $burntoolStateGood) { + $process = 0; +# $previousTable = "in=$outTableReal"; + $previousTableStyle = -1; + } + else { + $process = 1; + } + if ($REALRUN == 0) { + print "##Pretending to process as instructed!\n"; + $process = 1; + } + if (($process == 1)||($processNext == 1)) { + $processNext = 1; + # Set state to processing + my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + $file->{burntool_state} = -1; + + my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + # Run burntool + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + if ($previousTable ne '') { + $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN); + } + else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + + $status = vsystem ("$nebXattr --write $outTable user.copies:2"); + $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + + $previousTableStyle = -1; + + print "\n"; + } + + if ($previousTableStyle == 1) { + $previousTable = "infits=$rawImfileReal"; + } + elsif ($previousTableStyle == -1) { + $previousTable = "in=$outTableReal"; + } + else { + die "previousTableStyle undefined!\n"; + } + +} + +exit 0; + +sub vsystem { + my $command = shift; + my $realrun = shift; + + print "$command\n"; + + my $status = 0; + if ($realrun) { + $status = system ($command); + } + + return $status; +} + +sub my_die { + my $message = shift; + if ($#_ != -1) { + my $exp_id = shift; + my $class_id = shift; + vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1); + } + printf STDERR "$message\n"; + exit 1; +} Index: trunk/ippScripts/scripts/magic_process.pl =================================================================== --- trunk/ippScripts/scripts/magic_process.pl (revision 26831) +++ trunk/ippScripts/scripts/magic_process.pl (revision 26901) @@ -31,4 +31,5 @@ my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1); +my $VerifyStreaks = can_run('VerifyStreaks') or (warn "Can't find VerifyStreaks, will not produce png images"); if ($missing_tools) { warn("Can't find required tools."); @@ -37,5 +38,5 @@ # Parse the command-line arguments -my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile); +my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile); GetOptions( @@ -44,5 +45,5 @@ 'camera=s' => \$camera, # Camera name 'dbname=s' => \$dbname, # Database name - 'outroot=s' => \$outroot, # Output root name + 'baseroot=s' => \$baseroot, # Output root name 'save-temps' => \$save_temps, # Save temporary files? 'verbose' => \$verbose, # Print stuff? @@ -53,24 +54,30 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_id --camera --node --outroot", +pod2usage( -msg => "Required options: --magic_id --camera --node --baseroot", -exitval => 3) unless defined $magic_id and defined $node and defined $camera and - defined $outroot; + defined $baseroot; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $node, $PS_EXIT_SYS_ERROR ) if $logfile; -# DetectStreaks doesn't know about nebulous. It expects to be able to append strings to outroot -# to form valid file names. -# So forbid nebulous path in outroot. We could relax this by change DetectStreaks to take all -# of the file names as arguments or by teaching it about Nebulous -if ($outroot =~ 'neb:/') { +# DetectStreaks doesn't know about nebulous. It expects to be able to +# append strings to baseroot to form valid file names. So forbid +# nebulous path in baseroot. We could relax this by change +# DetectStreaks to take all of the file names as arguments or by +# teaching it about Nebulous +if ($baseroot =~ 'neb:/') { &my_die("DetectStreaks does not support nebulous paths in outroot", $magic_id, $node, $PS_EXIT_CONFIG_ERROR); } +# most filenames are of the form $baseroot.$node.*, but VerifyStreaks +# needs access to $baseroot.*, so we construct $outroot = +# $baseroot.$node in here + # resolve any path:// or file:// in outroot -$outroot = $ipprc->file_resolve($outroot); +$baseroot = $ipprc->file_resolve($baseroot); +my $outroot = "$baseroot.$node"; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -255,4 +262,6 @@ } + &run_verifystreaks($baseroot); + my $command = "$magictool -addmask"; $command .= " -magic_id $magic_id"; @@ -275,7 +284,54 @@ } - - ### Pau. + +sub run_verifystreaks { + + my $baseroot = shift; + + unless ($VerifyStreaks) { + print STDERR "skipping VerifyStreaks\n"; + return 1; + } + + # VerifyStreaks --out $outdir --clusters $outdir/clusters.list $rootname.root.streakMap + + my $outdir = "$baseroot.verify"; + + my($status) = system ("mkdir -p $outdir"); + if ($status) { + print STDERR "failed to create output directory $outdir\n"; + return 1; + } + + my $FILE; + + my @files = <$baseroot.*.clusters>; + + unless (open ($FILE, ">$outdir/clusters.list")) { + print "failed to create cluster file $outdir/clusters.list\n"; + return 1; + } + foreach my $file (@files) { + $file =~ s|.clusters$||; + print $FILE "$file\n"; + } + close ($FILE); + if ($status) { + print "failed to create cluster file $outdir/clusters.list\n"; + return 1; + } + + my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print "failed to run VerifyStreaks:\n"; + return 1; + } + + return 0; +} sub open_list_file { Index: trunk/ippScripts/scripts/make_burntool_pcontrol.pl =================================================================== --- trunk/ippScripts/scripts/make_burntool_pcontrol.pl (revision 26901) +++ trunk/ippScripts/scripts/make_burntool_pcontrol.pl (revision 26901) @@ -0,0 +1,459 @@ +#!/usr/bin/perl -w +# script to generate a pcontrol.pro file to run burntool on a set of data. + +use DBI; +use DateTime; +use Getopt::Std; + +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + +getopts('hbPQ:d:',\%opt); +if (exists($opt{h})) { + print STDERR "Usage: make_burntool_pcontrol.pl -b {-Q | -d | DATE_MIN0 DATE_MAX0 DATE_MIN1 DATE_MAX1 [...]}\n"; + print STDERR " -Q SQL_WHERE Generate pcontrol from a SQL WHERE query.\n"; + print STDERR " (enclose in single quotes, and use double quotes inside to\n"; + print STDERR " protect from the shell)\n"; + print STDERR " -d DATE Specify a single day to look for.\n"; + print STDERR " -b Only print burntool lines.\n"; + print STDERR " -P PR images have bad obs_mode values. Work around that.\n"; + print STDERR "\n"; + print STDERR " Scans the GPC1 database, and identifies \"science\" observations based on the obs_mode.\n"; + print STDERR " Use a 30 minute safety prelude before the first science data to make sure burntool works.\n"; + + exit(1); +} + +# Load database +$dbname = 'gpc1'; +$dbserver = 'ippdb01'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +# Determine what the value of "BURNTOOL.STATE.GOOD" currently is: +$burntoolStateGood = 999; +open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; +while() { + chomp; + if ($_ =~ /BURNTOOL.STATE.GOOD/) { + @line = split /\s+/; + $burntoolStateGood = $line[2]; + } +} +close(LAZY); +#unless( $burntoolStateGood == 999) { +# print STDERR "GOOD == $burntoolStateGood\n"; +#} + + +# List of which obs_modes are "science" and which are something else. +%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, + 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 1, ' ' => 0); +if (exists($opt{P})) { + $science{Unknown} = 1; +} + +# Zero the arrays and counters +@dates_min = (); +@dates_max = (); +$N_burntooled_images = 0; +$N_total_images = 0; +$N_ranges = 0; +# Parse SQL query and fill any valid dates to the arrays +if (exists($opt{Q})) { + $opt{Q} =~ s/\"/\'/g; + $sth = "SELECT dateobs FROM rawExp WHERE $opt{Q}"; + $data_ref = $db->selectall_arrayref( $sth ); + + %day_keys = (); + foreach $row_ref (@{ $data_ref }) { + ($day,$time) = split /\s+/, ${ $row_ref }[0]; + + unless (exists($day_keys{$day})) { + push @dates_min, $day; + + ($year,$mon,$date) = split /-/, $day; + $date++; + push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); + $day_keys{$day} = 1; +# print ">>>D $day $#dates_min\n"; + } + } +} + +# Read a single date, and calculate the end and insert +elsif (exists($opt{d})) { + push @dates_min, $opt{d}; + ($year,$mon,$date) = split /-/, $opt{d}; + $date++; + push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); +} + +# Just shift the supplied values into the arrays. +else { + if ($#ARGV == -1) { + system("$0 -h"); + exit(-1); + } + + while ($#ARGV > -1) { + push @dates_min, shift(@ARGV); + push @dates_max, shift(@ARGV); + } +} +unless(exists($opt{b})) { + print_prologue(); +} +# Save the query used in the pcontrol to be safe. +if (exists($opt{Q})) { + print "##query: SELECT dateobs FROM rawExp WHERE $opt{Q}\n"; +} + +while ($#dates_min > -1) { + $date_min = shift(@dates_min); + $date_max = shift(@dates_max); + + # Find _ALL_ observations within the date range we care about + $sth = "SELECT exp_id,dateobs,pon_time,exp_type,obs_mode,comment FROM rawExp WHERE " . + "dateobs >= '${date_min}' AND dateobs <= '${date_max}' order by dateobs"; + + $data_ref = $db->selectall_arrayref( $sth ); + + $seq = 0; + + # Identify the science observations, and make a window earlier for burntool. + @start_s = (0); + @end_s = (0); + @skips = (0); + + foreach $row_ref (@{ $data_ref }) { + + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + # Fix nulls in the database + if (!defined($obs_mode)) { + $obs_mode = ' '; + } + if (!defined($comment)) { + $comment = ' '; + } + + ($date,$time) = split / /, $dateobs; + ($year,$month,$day) = split /-/, $date; + ($hour,$minute,$second) = split /:/, $time; + + $dt = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu'); + + $st = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu')->subtract(minutes => 30); + $et = $dt->epoch(); + + + if (!exists($science{$obs_mode})) { + $science{$obs_mode} = 0; + } + if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| + (($science{$obs_mode} == 0)&&( + ($comment =~ /MD/)||($comment =~ /ThreePi/)|| + ($comment =~ /STS/)||($comment =~ /M31/)|| + ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)|| + ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)|| + ($comment =~ /sdss/)))) + { + if ($start_s[-1] == 0) { + $start_s[-1] = $et - 1800; + $end_s[-1] = $et + 1; + } + else { + if ($et > $end_s[-1] + 1800) { + push @start_s, $et - 1800; + push @end_s, $et + 1; + push @skips, 0; + } + else { + $end_s[-1] = $et + 1; + } + } + } + + $seq++; + + } + + # If this night has no science exposures, skip to the next one. + if (($#start_s == 0) && ($start_s[0] == 0)) { + next; + } + + # Scan again and count how many exposures are between windows + foreach $row_ref (@{ $data_ref }) { + ($exp_id,$dateobs,$pontime,$exp_type,$obs_mode,$comment) = @{ $row_ref }; + if (!defined($obs_mode)) { + $obs_mode = ' '; + } + if (!defined($comment)) { + $comment = ' '; + } + + ($date,$time) = split / /, $dateobs; + ($year,$month,$day) = split /-/, $date; + ($hour,$minute,$second) = split /:/, $time; + + $dt = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu'); + + $st = DateTime->new( year => $year, month => $month, day => $day, + hour => $hour, minute => $minute, second => $second, + nanosecond => 0, time_zone => 'Pacific/Honolulu')->subtract(minutes => 30); + $et = $dt->epoch(); + + if (!exists($science{$obs_mode})) { + die "No mode >>$obs_mode<<\n"; + } + + for ($i = 0; $i <= $#start_s; $i++) { + if (($et < $start_s[$i])&&($science{$obs_mode} == 0)) { + if ($i == 0) { + $skips[$i]++; + } + else { + if ($et > $end_s[$i-1]) { + $skips[$i]++; + } + } + } + } + } + + # Clear out windows that overlap with previous ones so we have the minimum set + for ($i = $#start_s; $i > 0; $i--) { + if ($skips[$i] == 0) { + $trash = pop(@start_s); + $end = pop(@end_s); + $trash = pop(@skips); + $end_s[-1] = $end; + } + } + # Print out the end points in the format burntool wants. + + for ($i = 0; $i <= $#start_s; $i++) { + @start_t = localtime($start_s[$i]); + @end_t = localtime($end_s[$i]); + + my $date_min = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", + $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], + $start_t[2],$start_t[1],$start_t[0]); + my $date_max = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", + $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], + $end_t[2],$end_t[1],$end_t[0]); + + my $total_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max'"; + my $im_ref = $db->selectall_arrayref( $total_images_sth ); + $N_total_images += ${ $im_ref }[0][0]; + + my $burntooled_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max' AND abs(burntool_state) = $burntoolStateGood"; + $im_ref = $db->selectall_arrayref( $burntooled_images_sth ); + $N_burntooled_images += ${ $im_ref }[0][0]; + + + printf(" burntool %s %s\n",$date_min,$date_max); + } + unless(exists($opt{d})) { + print "\n"; + } + $N_ranges += ($#start_s + 1); +} + +if ($N_ranges == 0) { + print_short_epilogue(); +} + +unless(exists($opt{b}) || ($N_ranges == 0)) { + print_epilogue(); +} + +## +## These functions print out the remaining text of the pcontrol.pro script. I've removed things +## that seemed superfluous. +## + +sub print_prologue { + + print << 'END_PROLOGUE'; +# this script sets up a series of burntool runs targetted to the appropriate machine + +# use the following sql to get the host table match: +# select exp_id, exp_name, class_id, dateobs, user_1, obs_mode, uri from rawImfile where exp_id = 33750 limit 100; + +macro burntool + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + list word -split $hostmatch:$i + $class_id = $word:0 + $logfile = "burntool_logs/$class_id.$1.log" + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile + end +end + +macro go +END_PROLOGUE + +return(0); +} +sub print_short_epilogue { + if (exists($opt{b})) { + print STDERR "There were no science exposures to process. Sorry, try again tomorrow.\n"; + return(); + } + print << 'END_SHORT_EPILOGUE'; + + echo "There were no science exposures to process. Sorry, try again tomorrow." + +end +END_SHORT_EPILOGUE +} + +sub print_epilogue { + print " echo 'There were $N_total_images total images, of which $N_burntooled_images were already burntooled.'\n"; + print "### BTSTAT $N_total_images $N_burntooled_images\n"; + print << 'END_EPILOGUE'; + +end + +macro setnames + $burntool_range:20081001 = 2008-10-01T07:50:00 2008-10-01T15:05:00 +end + +# for a re-run add --skip_burned: +# job -host $word:1 ipp_apply_burntool.pl --class_id $word:0 --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --skip_burned + +macro loadhosts + for i 0 $allhosts:n + host add $allhosts:$i + end +end + +# this macro may be used to complete a burntool run that exited due to error +macro burntool_skip_burned + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + list word -split $hostmatch:$i + $class_id = $word:0 + $logfile = "burntool_logs/$class_id.$1.log" + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --skip_burned --logfile $logfile + end +end + +list hostmatch + XY01 ipp014 + XY02 ipp014 + XY03 ipp038 + XY04 ipp038 + XY05 ipp023 + XY06 ipp023 + XY10 ipp039 + XY11 ipp039 + XY12 ipp024 + XY13 ipp024 + XY14 ipp040 + XY15 ipp040 + XY16 ipp026 + XY17 ipp026 + XY20 ipp041 + XY21 ipp041 + XY22 ipp042 + XY23 ipp042 + XY24 ipp043 + XY25 ipp043 + XY26 ipp028 + XY27 ipp028 + XY30 ipp044 + XY31 ipp044 + XY32 ipp029 + XY33 ipp029 + XY34 ipp045 + XY35 ipp045 + XY36 ipp030 + XY37 ipp030 + XY40 ipp046 + XY41 ipp046 + XY42 ipp031 + XY43 ipp031 + XY44 ipp047 + XY45 ipp047 + XY46 ipp032 + XY47 ipp032 + XY50 ipp048 + XY51 ipp048 + XY52 ipp033 + XY53 ipp033 + XY54 ipp049 + XY55 ipp049 + XY56 ipp034 + XY57 ipp034 + XY60 ipp050 + XY61 ipp050 + XY62 ipp035 + XY63 ipp035 + XY64 ipp051 + XY65 ipp051 + XY66 ipp036 + XY67 ipp036 + XY71 ipp052 + XY72 ipp052 + XY73 ipp015 + XY74 ipp015 + XY75 ipp053 + XY76 ipp053 +end +list allhosts + ipp043 + ipp014 + ipp015 + ipp023 + ipp024 + ipp026 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end + +END_EPILOGUE +return(0); +}