Index: /trunk/ippScripts/scripts/automate_stacks.pl =================================================================== --- /trunk/ippScripts/scripts/automate_stacks.pl (revision 26620) +++ /trunk/ippScripts/scripts/automate_stacks.pl (revision 26620) @@ -0,0 +1,591 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Set up +my $missing_tools = 0; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# global variables. +my @target_list = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10'); +my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); +my $db; +my $debug = 0; +my %metadata_out = (); +$metadata_out{nsState} = 'UNDEFINEDERROR'; +my $loghead; +chomp($loghead = `date`); +print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n"; +# Grab options +my ( $date, $camera, $dbname, $logfile, $verbose); +my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips, $check_stacks, $queue_stacks); +my ( $help, $this_target_only, $this_filter_only, $isburning); +GetOptions( + 'help|h' => \$help, + 'date=s' => \$date, # night to make stacks for + 'camera=s' => \$camera, # camera + 'dbname=s' => \$dbname, # Database name + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, + 'check_chips' => \$check_chips, + 'queue_chips' => \$queue_chips, + 'check_stacks' => \$check_stacks, + 'queue_stacks' => \$queue_stacks, + 'queue_burntool' => \$queue_burntool, + 'define_burntool' => \$define_burntool, + 'check_registration|R' => \$check_registration, + 'this_target_only=s' => \$this_target_only, + 'this_filter_only=s' => \$this_filter_only, + 'isburning' => \$isburning, + ) or pod2usage ( 2 ); +pod2usage( -msg => +"USAGE: automate_stacks.pl + Options: + --help This help. + --date YYYY-MM-DD Work on this date (defaults to today GMT). + --camera Default GPC1. + --dbname Default gpc1. + --verbose + --this_target_only Process only a single target. + --this_filter_only Process only a single filter. + Modes: + --check_registration Confirm the data downloaded correctly. + --define_burntool Determine date ranges for burntool. + --queue_burntool Issue commands to burntool data. + --check_chips Confirm that chips can be built. + --queue_chips Issue chiptool commands to queue chips. + --check_stacks Confirm that stacks can be built. + --queue_stacks Issue stacktool commands to queue stacks.\n", + -exitval => 2, ) if (defined($help)); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks", + -exitval => 3, + ) unless + defined $check_registration or defined $define_burntool or defined $queue_burntool or + defined $queue_chips or defined $queue_stacks or + defined $check_chips or defined $check_stacks; + + +unless(defined($date)) { + my @trash = gmtime; + $trash[5] += 1900; + $trash[4] += 1; + $date = sprintf("%4d-%02d-%02d",$trash[5],$trash[4],$trash[3]); +} +unless(defined($camera)) { + $camera = 'GPC1'; +} +unless(defined($dbname)) { + $dbname = 'gpc1'; +} + +if (defined($this_target_only)) { + foreach my $t (@target_list) { + if ($t eq $this_target_only) { + @target_list = ($this_target_only); + last; + } + } + die("$this_target_only is invalid.") if ($#target_list != 0); +} + +if (defined($this_filter_only)) { + foreach my $t (@filter_list) { + if ($t eq $this_filter_only) { + @filter_list = ($this_filter_only); + last; + } + } + die("$this_filter_only is invalid.") if ($#filter_list != 0); +} + +if (defined($check_registration)) { + my ($Nsummit_exp,$Nfaults) = check_summit_copy($date); + + if ($Nsummit_exp == 0) { + print STDERR "No exposures were found on the summit for $date.\n"; + $metadata_out{nsState} = 'DROP'; + } + elsif ($Nfaults != 0) { + print STDERR "There were faults while downloading the exposures for $date.\n"; + $metadata_out{nsState} = 'NEW'; + } + else { + print STDERR "Summit copy and Registration have succeeded for $date.\n"; + $metadata_out{nsState} = 'REGISTERED'; + } + return_metadata($date); + exit(0); +} +if (defined($define_burntool)) { + $metadata_out{nsState} = 'QUEUEBURNING'; + &find_burntool_ranges($date); + return_metadata($date); + exit(0); +} +if (defined($queue_burntool)) { + $metadata_out{nsState} = 'BURNING'; + return_metadata($date); + exit(0); +} +if (defined($check_chips)) { + $metadata_out{nsState} = 'QUEUECHIPS'; + &execute_chips($date,"pretend"); + if (($metadata_out{nsState} eq 'NEEDSBURNING')&&(defined($isburning))) { + $metadata_out{nsState} = 'BURNING'; + } + return_metadata($date); + exit(0); +} +if (defined($check_stacks)) { + $metadata_out{nsState} = 'QUEUESTACKS'; + &execute_stacks($date,"pretend"); + return_metadata($date); + exit(0); +} +if (defined($queue_chips)) { + $metadata_out{nsState} = 'TOWARP'; + &execute_chips($date); + return_metadata($date); + exit(0); +} +if (defined($queue_stacks)) { + $metadata_out{nsState} = 'STACKING'; + &execute_stacks($date); + return_metadata($date); + exit(0); +} + +# Registration + +sub check_summit_copy { + my $date = shift; + my $db = init_gpc_db(); + + # largely stolen from Bill's checkexp program. + + my $sth = " SELECT exp_name, summitExp.dateobs AS registered, rawExp.dateobs, summitExp.imfiles, "; + $sth .= " summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, "; + $sth .= " count(pzDownloadImfile.class_id) AS download_count, newExp.state AS newExp_state, newExp.exp_id, summitExp.exp_type"; + $sth .= " FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) "; + $sth .= " LEFT JOIN newExp ON exp_name = tmp_exp_name LEFT JOIN rawExp USING(exp_id, exp_name) "; + $sth .= " WHERE date(summitExp.dateobs) >= '${date}T00:00:00' AND date(summitExp.dateobs) <= '${date}T23:59:59' "; + $sth .= " GROUP BY exp_name ORDER BY summitExp.dateobs "; + + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nsummit_exps = 0; + my $Nsummit_faults = 0; + my $Ndownload_faults = 0; + my $Nregister_faults = 0; + +# my_trace($sth,$data_ref,$#{ $data_ref }); + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault, + $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref }; + $Nsummit_exps++; + if ($summit_fault) { + print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; + if ($exp_type ne 'OBJECT') { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nsummit_faults++; + } + } + elsif (!$download_state or $download_state eq 'run') { + print STDERR "check_summit_copy: $date $exp_name has download_state $download_state"; + if ($exp_type ne 'OBJECT') { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Ndownload_faults++; + } + } + elsif (!$new_state or $new_state eq 'run' ) { + print STDERR "check_summit_copy: $date $exp_name has new_state $new_state"; + if ($exp_type ne 'OBJECT') { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nregister_faults++; + } + } + } + + return($Nsummit_exps,$Nsummit_faults + $Ndownload_faults + $Nregister_faults); +} + +# Burntool + +sub get_goodBTvalue { + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR); + my $burntoolStateGood = 999; + foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } + } + if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR); + } + return($burntoolStateGood); +} + +sub verify_burntool { + my $exp_id = shift; + my $burntoolStateGood = shift; + + my $db = init_gpc_db(); + + my $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nimfiles = 0; + my $Nburntooled = 0; + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref }; + $Nimfiles++; + if (abs($burntool_state) == $burntoolStateGood) { + $Nburntooled++; + } + } + return($Nimfiles,$Nburntooled); +} + +sub find_burntool_ranges { + my $date = shift; + # Much cleaner than reimplementing that mess here + my $command = "$mkBTpcontrol -d $date -b"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + if ($#{ $stderr_buf } > -1) { + $metadata_out{nsState} = 'ERROR'; + return(1); + } + + my $N = 0; + my @burntool_entries = split /\n/, (join '', @$stdout_buf); + foreach my $row (@burntool_entries) { + my ($trash,$start,$end); + ($trash,$trash,$start,$end) = (split /\s+/,$row); + $metadata_out{"btBegin${N}"} = $start; + $metadata_out{"btEnd${N}"} = $end; + $metadata_out{"btCounter${N}"} = 0; + $N++; + print STDERR "define_burntool: $row\n"; + } + $metadata_out{btN} = $N - 1; + $metadata_out{btNCounter} = 0; + + return(0); +} + + +# Chips + +sub construct_chiptool_cmd { + my $date = shift; + my $target = shift; + + my $select = "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$chiptool"; + $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp '; + $cmd .= " -set_label ${target}.nightlystack "; + $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target "; + $cmd .= " -set_tess_id $target -comment ${target}% -set_data_group ${target}.${date} "; + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= " -pretend "; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_chip { + my $exp_id = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my $data_group = "${target}.${date}"; + + my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_chip_queue { + my $date = shift; + my $target = shift; + + my $command = construct_chiptool_cmd($date,$target) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + my $burntoolStateGood = get_goodBTvalue(); + my $Nimfiles = 0; + my $Nburntooled = 0; + my $Nalready = 0; + my @input_exposures = split /\n/, (join '', @$stdout_buf); + foreach my $entry (@input_exposures) { + my ($exp_id, @trash) = split /\s+/, $entry; + @trash = verify_burntool($exp_id,$burntoolStateGood); + $Nimfiles += $trash[0]; + $Nburntooled += $trash[1]; + + @trash = verify_uniqueness_chip($exp_id,$date,$target); + $Nalready += $trash[0]; + } + return($#input_exposures + 1,$Nimfiles,$Nburntooled,$Nalready); +} + +sub chip_queue { + my $date = shift; + my $target = shift; + + my $command = construct_chiptool_cmd($date,$target); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $metadata_out{nsState} = 'TOWARP'; + return(0); +} + +sub execute_chips { + my $date = shift; + my $pretend = shift; + foreach my $target (@target_list) { + my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_chips: Target $target on $date had no exposures.\n"; + next; + } + if ($Nalready != 0) { + print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n"; + next; + } + if ($Nimfiles != $Nburntooled) { + print STDERR "execute_chips: Target $target on $date is not fully burntooled.\n"; + $metadata_out{nsState} = 'NEEDSBURNING'; + next; + } + unless(defined($pretend)) { + chip_queue($date,$target); + } + } +} + +# Stacking + +sub construct_stacktool_cmd { + my $date = shift; + my $target = shift; + my $filter = shift; + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$stacktool"; +# $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite + $cmd .= ' -simple -dbname gpc1 -definebyquery '; + $cmd .= " -set_label ${target}.nightlystack -select_label ${target}.nightlystack "; + $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target "; + $cmd .= " -select_filter $filter -set_data_group ${target}.${date} "; + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_stack { + my $skycell = shift; + my $date = shift; + my $target = shift; + my $filter = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my $data_group = "${target}.${date}"; + + my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_stack_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + + # check warp stage == chip stage + my $db = init_gpc_db(); + + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $where = " label = '${target}.nightlystack' AND data_group = '${target}.${trunc_date}' "; + my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; + my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; + + my $chip_ref = $db->selectall_arrayref( $chip_sth ); + my $warp_ref = $db->selectall_arrayref( $warp_sth ); + + # check that we will be able to queue them up + my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR); + } + + my $Nalready = 0; + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + foreach my $entry (@input_exposures) { + my ($skycell, @trash) = split /\s+/, $entry; + @trash = verify_uniqueness_stack($skycell,$date,$target,$filter); + $Nalready += $trash[0]; + } + + return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $warp_ref } + 1, $Nalready); +} + +sub stack_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + + my $command = construct_stacktool_cmd($date,$target,$filter); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); +} + +sub execute_stacks { + my $date = shift; + my $pretend = shift; + foreach my $target (@target_list) { + foreach my $filter (@filter_list) { + my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); + if (($NprocChips != $NprocWarps)||($NprocWarps == 0)||($NprocChips == 0)) { + print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + $metadata_out{nsState} = 'TOWARP'; + next; + } + if ($Nexposures == 0) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + next; + } + if ($Nalready != 0) { + print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + next; + } + unless(defined($pretend)) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + stack_queue($date,$target,$filter); + } + } + } + +} + +# Utilities + +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} + +sub return_metadata { + my $date = shift; + print STDOUT "autoStack MULTI\n\n"; + print STDOUT "autoStack METADATA\n"; + print STDOUT " date STR $date\n"; + foreach my $k (keys %metadata_out) { + print STDOUT " $k STR $metadata_out{$k}\n"; + } + print STDOUT "END\n"; +} + +sub my_die { + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} + +sub my_trace { + if ($debug == 1) { + foreach my $thing (@_) { + carp($thing); + } + } +} Index: /trunk/ippTasks/nightly_stacks.pro =================================================================== --- /trunk/ippTasks/nightly_stacks.pro (revision 26620) +++ /trunk/ippTasks/nightly_stacks.pro (revision 26620) @@ -0,0 +1,688 @@ +## nightly_stacks.pro : -*- sh -*- + +check.globals + +macro nightly.stacks.on + ns.initday.on + ns.registration.on + ns.burntool.on + ns.chips.on + ns.stacks.on +end + +macro nightly.stacks.off + ns.initday.off + ns.registration.off + ns.burntool.off + ns.chips.off + ns.stacks.off +end + + + +macro ns.initday.on + task ns.initday.load + active true + end +end + +macro ns.registration.on + task ns.registration.load + active true + end +end + +macro ns.burntool.on + task ns.burntool.load + active true + end + task ns.burntool.run + active true + end +end + +macro ns.chips.on + task ns.chips.load + active true + end + task ns.chips.run + active true + end +end + +macro ns.stacks.on + task ns.stacks.load + active true + end + task ns.stacks.run + active true + end +end + +macro ns.initday.off + task ns.initday.load + active false + end +end + +macro ns.registration.off + task ns.registration.load + active false + end +end + +macro ns.burntool.off + task ns.burntool.load + active false + end + task ns.burntool.run + active false + end +end + +macro ns.chips.off + task ns.chips.load + active false + end + task ns.chips.run + active false + end +end + +macro ns.stacks.off + task ns.stacks.load + active false + end + task ns.stacks.run + active false + end +end + +$ns_regPAGE = 0 +$ns_burnPAGE = 0 +$ns_RburnPAGE = 0 +$ns_RburnCELL = 0 +$ns_chipPAGE = 0 +$ns_RchipPAGE = 0 +$ns_stackPAGE = 0 +$ns_RstackPAGE = 0 + +book init nsData + +# +# Start a new date to work on +# +macro ns.add.date + book newpage nsData $1 + book setword nsData $1 nsState NEW +end + +task ns.initday.load + host local + periods -poll 86400 + periods -exec 86400 + periods -timeout 30 + npending 1 + + task.exec + $today = `date +%Y-%m-%d` + book newpage nsData $today + book setword nsData $today nsState NEW + + command echo $today + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if registration is complete +# +task ns.registration.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.registration.log + + book getpage nsData $ns_regPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages + end + + $ns_regPAGE ++ + if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 + option $date + + if ("$ns_STATE" != "NEW") break + $run = automate_stacks.pl --check_registration --date $date + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the chips have been properly burntooled and are ready for queueing +# +task ns.chips.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.chips.log + + book getpage nsData $ns_chipPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages + end + + $ns_chipPAGE ++ + if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 + option $date + + if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break + $run = automate_stacks.pl --check_chips --date $date + + if ("$ns_STATE" == "BURNING") + $run = $run --isburning + end + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the chips have been properly burntooled and are ready for queueing +# +task ns.chips.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.chips.log + + book getpage nsData $ns_RchipPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages + end + + $ns_RchipPAGE ++ + if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 + option $date + + if ("$ns_STATE" != "QUEUECHIPS") break + $run = automate_stacks.pl --queue_chips --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +# +# Check to see if the warps are finished +# +task ns.stacks.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_stackPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages + end + + $ns_stackPAGE ++ + if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + $run = automate_stacks.pl --check_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make stacks. +# +task ns.stacks.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_RstackPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.run: " $ns_RstackPAGE $date $ns_STATE $Npages + end + + $ns_RstackPAGE ++ + if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 + option $date + + if ("$ns_STATE" != "QUEUESTACKS") break + $run = automate_stacks.pl --queue_stacks --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Generate a list of date ranges that need to be processed with burntool +# +task ns.burntool.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.burntool.log + + book getpage nsData $ns_burnPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages + end + + $ns_burnPAGE ++ + if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 + option $date + + if ("$ns_STATE" != "NEEDSBURNING") break + $run = automate_stacks.pl --define_burntool --date $date + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Magically run burntool on the data, based on the time ranges stored in our book. +# +task ns.burntool.run + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 10800 + npending 20 + task.exec + stdout NULL + stderr $LOGDIR/ns.burntool.log + + book getpage nsData $ns_RburnPAGE -var date + book getword nsData $date nsState -var ns_STATE + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages + end + + $ns_RburnPAGE ++ + if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 + + if ("$ns_STATE" != "QUEUEBURNING") break + book getword nsData $date btN -var btN + book getword nsData $date btNCounter -var btNcounter + + if ($btNcounter > $btN) + book setword nsData $date nsState "BURNING" + break + end + + sprintf start_date_label "btBegin%d" $btNcounter + sprintf end_date_label "btEnd%d" $btNcounter + sprintf counter_label "btCounter%d" $btNcounter + + book getword nsData $date $start_date_label -var start_date + book getword nsData $date $end_date_label -var end_date + book getword nsData $date $counter_label -var chip_counter + + list word -split $hostmatch:$chip_counter + $class_id = $word:0 + $host = $word:1 + host $host + $logfile = "burntool_logs/$class_id.$start_date.log" + + $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile + + echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run + $chip_counter ++ + if ($chip_counter >= $hostmatch:n) + $chip_counter = 0 + $btNcounter ++ + end + $new_state = "QUEUEBURNING" + if ($btNcounter > $btN) + $new_state = "BURNING" + end + + echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + book setword nsData $date $counter_label $chip_counter + book setword nsData $date btNCounter $btNcounter + option $date $new_state + + command $run +# command /bin/true + end + # success + task.exit 0 +# convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + book setword nsData $options:0 nsState $options:1 + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +# +# This is all burntool related stuff. +# + +macro burntool + if ($0 != 3) + echo "USAGE: burntool (dateobs_begin) (dateobs_end)" + break + end + + for i 0 $hostmatch:n + job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile + end +end +macro loadhosts + for i 0 $allhosts:n + controller host add $allhosts:$i + end +end + +# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +list hostmatch + XY01 ipp014 + XY03 ipp038 + XY05 ipp023 + XY10 ipp039 + XY12 ipp024 + XY15 ipp040 + XY17 ipp020 + XY21 ipp041 + XY23 ipp042 + XY25 ipp043 + XY27 ipp028 + XY31 ipp044 + XY33 ipp029 + XY35 ipp045 + XY37 ipp030 + XY41 ipp046 + XY43 ipp031 + XY45 ipp047 + XY47 ipp032 + XY51 ipp048 + XY53 ipp033 + XY55 ipp049 + XY57 ipp034 + XY61 ipp050 + XY63 ipp035 + XY65 ipp051 + XY67 ipp036 + XY72 ipp052 + XY74 ipp015 + XY76 ipp053 + XY02 ipp014 + XY04 ipp038 + XY06 ipp023 + XY11 ipp039 + XY13 ipp024 + XY14 ipp040 + XY16 ipp020 + XY20 ipp041 + XY22 ipp042 + XY24 ipp043 + XY26 ipp028 + XY30 ipp044 + XY32 ipp029 + XY34 ipp045 + XY36 ipp030 + XY40 ipp046 + XY42 ipp031 + XY44 ipp047 + XY46 ipp032 + XY50 ipp048 + XY52 ipp033 + XY54 ipp049 + XY56 ipp034 + XY60 ipp050 + XY62 ipp035 + XY64 ipp051 + XY66 ipp036 + XY71 ipp052 + XY73 ipp015 + XY75 ipp053 +end + +list allhosts + ipp043 + ipp014 + ipp015 + ipp023 + ipp024 + ipp020 + ipp028 + ipp029 + ipp030 + ipp031 + ipp032 + ipp033 + ipp034 + ipp035 + ipp036 + ipp038 + ipp039 + ipp040 + ipp041 + ipp042 + ipp044 + ipp045 + ipp046 + ipp047 + ipp048 + ipp049 + ipp050 + ipp051 + ipp052 + ipp053 +end Index: /trunk/tools/wiki_nightly_stacks_table.pl =================================================================== --- /trunk/tools/wiki_nightly_stacks_table.pl (revision 26620) +++ /trunk/tools/wiki_nightly_stacks_table.pl (revision 26620) @@ -0,0 +1,284 @@ +#! /usr/local/bin/perl -w + +use DBI; +use Getopt::Std; + +getopts('P',\%opt); + +$dbname = 'gpc1'; +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} + +$db = init_gpc_db(); + +@targets = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10'); + +# potential targets +if (exists($opt{P})) { + foreach $t (@targets) { + $sth = "select filter,DATE_FORMAT(rawExp.dateobs,'%Y-%m-%d') AS day,count(filter) FROM rawExp JOIN newExp using(exp_id) WHERE rawExp.fault = 0 AND exp_type = 'OBJECT' AND comment LIKE '${t}%' AND dateobs >= '2009-12-08T00:00:00' GROUP BY day,filter"; + $data_ref = $db->selectall_arrayref( $sth ); + foreach $rr (@{ $data_ref }) { + ($filter,$date,$N) = @{ $rr }; + $filter =~ s/.00000//; + $potential{$date}{$t}{$filter} = $N; + } + } + print "|| '''Potential (Nexp):'''\n||\n"; + print '|| Date ||'; + foreach $t (@targets) { + print "$t ||"; + } + print "\n"; + + foreach $d (sort (keys %potential)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $potential{$d}{$t} })) { + if ($i == 0) { + print "${f}($potential{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($potential{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; + } + print "||\n"; +} +#queued in chip + +$sth = "select rawExp.dateobs,chipRun.state,rawExp.filter,count(chipRun.state),chipRun.data_group,chipRun.dist_group from chipRun JOIN rawExp using (exp_id) where chipRun.label LIKE '%nightlystack' GROUP BY data_group,filter,state ORDER BY dateobs"; + +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($date,$state,$filter,$N,$data_group,$dist_group) = @{ $rr }; + if ($state eq 'full') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $chips{$date}{$dist_group}{$filter} = $N; + } + if ($state eq 'new') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $queued{$date}{$dist_group}{$filter} = $N; + } + +} +print "|| '''Queued for Chip (Nexp):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %queued)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $queued{$d}{$t} })) { + if ($i == 0) { + print "${f}($queued{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($queued{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; +print "|| '''Through Chip (Nexp):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %chips)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $chips{$d}{$t} })) { + if ($i == 0) { + print "${f}($chips{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($chips{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +#warps +$sth = "select rawExp.dateobs,warpRun.state,rawExp.filter,count(warpRun.state),warpRun.data_group,warpRun.dist_group from warpRun JOIN fakeRun using (fake_id) JOIN camRun using (cam_id) JOIN chipRun using (chip_id) JOIN rawExp using (exp_id) where warpRun.label LIKE '%nightlystack' GROUP BY data_group,filter ORDER BY data_group,filter"; +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($date,$state,$filter,$N,$data_group,$dist_group) = @{ $rr }; + if ($state eq 'full') { + ($date,$trash) = split /\s+/,$date; + $filter =~ s/.00000//; + $data{$date}{$dist_group}{$filter} = $N; + } +} + +print "|| '''Through Warp (Nexp):'''\n||\n"; + +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %data)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $data{$d}{$t} })) { + if ($i == 0) { + print "${f}($data{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($data{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +# stacks + + +$sth = "select state,count(skycell_id),data_group,dist_group,filter from stackRun where label LIKE '%nightlystack' group by data_group,filter,state"; +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($state,$N,$data_group,$dist_group,$filter) = @{ $rr }; + $filter =~ s/.00000//; + if ($state eq 'full') { + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + $stacks{$date}{$dist_group}{$filter} = $N; + } + if ($state eq 'new') { + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + $queuestacks{$date}{$dist_group}{$filter} = $N; + } +} +print "|| '''Queued Stack (Nskycells):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %queuestacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $queuestacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($queuestacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($queuestacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; +print "|| '''Through Stack (Nskycells):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %stacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $stacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($stacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($stacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "||\n"; + +# distribution + +$sth = "select distRun.data_group,dist_group,filter,count(stage_id) from distRun JOIN stackRun ON (stage_id = stack_id) where distRun.label LIKE '%.nightlystack' AND distRun.state = 'full' GROUP BY distRun.data_group,filter"; + +$data_ref = $db->selectall_arrayref( $sth ); + +foreach $rr (@{ $data_ref }) { + ($data_group,$dist_group,$filter,$N) = @{ $rr }; + $filter =~ s/.00000//; + ($trash,$date_str) = split /\./, $data_group; + $date = sprintf("%d%d%d%d-%d%d-%d%d",(split //,$date_str)); + + $diststacks{$date}{$dist_group}{$filter} = $N; +} +print "|| '''Distributed (Nstack_ids):'''\n||\n"; +print '|| Date ||'; +foreach $t (@targets) { + print "$t ||"; +} +print "\n"; + +foreach $d (sort (keys %diststacks)) { + print '||' . $d ; + foreach $t (@targets) { + print '|| ' ; + $i = 0; + foreach $f (sort (keys %{ $diststacks{$d}{$t} })) { + if ($i == 0) { + print "${f}($diststacks{$d}{$t}{$f})"; + $i++; + } + else { + print ",${f}($diststacks{$d}{$t}{$f})"; + } + } + } + print '||' . "\n"; +} +print "\n\n"; +