Index: /branches/eam_branches/ipp-20101205/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/Build.PL (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/Build.PL (revision 30102) @@ -105,6 +105,8 @@ scripts/whichimage scripts/automate_stacks.pl + scripts/nightly_science.pl scripts/lossy_compress_imfile.pl scripts/ipp_apply_burntool.pl + scripts/ipp_apply_burntool_single.pl scripts/make_burntool_pcontrol.pl scripts/bundle_detrends.pl Index: /branches/eam_branches/ipp-20101205/ippScripts/MANIFEST =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/MANIFEST (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/MANIFEST (revision 30102) @@ -37,6 +37,8 @@ scripts/ds9_cmf_regions.pl scripts/automate_stacks.pl +scripts/nightly_science.pl scripts/lossy_compress_imfile.pl scripts/ipp_apply_burntool.pl +scripts/ipp_apply_burntool_single.pl scripts/make_burntool_pcontrol.pl scripts/bundle_detrends.pl Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/automate_stacks.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/automate_stacks.pl (revision 30102) @@ -1355,5 +1355,5 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); - if ($target eq 'OSS') { + if (($target eq 'OSS')||($target eq 'SweetSpot')) { my $db = init_gpc_db(); Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/camera_exp.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/camera_exp.pl (revision 30102) @@ -378,5 +378,6 @@ $ipprc->file_exists($file); - if ($replicate and (file_scheme($file) eq 'neb')) { + my $scheme = file_scheme($file); + if ($replicate and $scheme and (file_scheme($file) eq 'neb')) { $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $cam_id, $PS_EXIT_SYS_ERROR); } Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/chip_imfile.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/chip_imfile.pl (revision 30102) @@ -176,4 +176,7 @@ $ipprc->delete_destreak_backup_file($outputWeight) or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR); + + # don't do binned images when updating unless we are starting from scratch. + $do_binned_images = 0 unless $dump_config; } if ($do_binned_images) { @@ -490,5 +493,7 @@ check_output($configuration, 1) if $dump_config; check_output($backmdl, 1) if $outputBackmdlExpect; - check_output($pattern, 1) if $outputPatternExpect; + # allow the pattern file to be missing if run state is update older data doesn't have one + # I should parse the config dump file to calculate the 'Expect' variables + check_output($pattern, 1, $run_state eq 'update') if $outputPatternExpect; if ($do_photom) { check_output($outputSources, 1); @@ -701,4 +706,5 @@ my $file = shift; my $replicate = shift; + my $allow_missing = shift; if (!defined $file) { @@ -706,5 +712,13 @@ } - &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + my $exists = $ipprc->file_exists($file); + + if (!$exists) { + if ($allow_missing) { + carp("Couldn't find expected output_file: $file but continuing anyways\n"); + return 1; + } + &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } if ($replicate and $neb) { Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_bundle.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_bundle.pl (revision 30102) @@ -62,4 +62,6 @@ my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $file_cmd = can_run('file') or (warn "can't find program file" and $missing_tools = 1); +my $zcat = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -526,8 +528,42 @@ $PS_EXIT_CONFIG_ERROR) if (!$resolved); + &my_die("config dump file resolved but not accessible: $config_file_rule", $component, + $PS_EXIT_CONFIG_ERROR) if !$ipprc->file_exists($resolved); + + my $mdc_compressed; + { + my $command = "$file_cmd $resolved"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + my $output = join "", @$stdout_buf; + # XXX: may need to to make this more robust + $mdc_compressed = ($output =~ /gzip/); + } + my $inName; + if ($mdc_compressed) { + my $tmpfile; + ($tmpfile, $inName) = tempfile( "/tmp/bundle.XXXX", UNLINK => !$save_temps ); + close($tmpfile) or &my_die("failed to close $inName", $component, $PS_EXIT_UNKNOWN_ERROR); + + my $command = "$zcat $resolved > $inName"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + + + } else { + $inName = $resolved; + } + my $in = open_with_retries($inName); + # we don't use the mdc parser because the perl parser is way is too slow for complicated config # files like this - my $in = open_with_retries($resolved); - my $line; while ($line = <$in>) { Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_apply_burntool_single.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30102) @@ -0,0 +1,187 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +#use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $REALRUN = 1; + +my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun ); +GetOptions( + 'exp_id=s' => \$exp_id, # exposure identifier + 'class_id=s' => \$class_id, # chip identifier + 'this_uri=s' => \$this_uri, # uri for this image + 'previous_uri=s' => \$previous_uri, # uri for the previous image + 'imfile_state=s' => \$imfile_state, # current state of the imfile. + 'camera=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'logfile=s' => \$logfile, + 'rerun' => \$rerun, + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --class_id --this_uri --dbname", + -exitval => 3, + ) unless + defined $class_id and + defined $this_uri and + defined $dbname; + +unless (defined $camera) { + $camera = "GPC1"; +} + +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $funpack = can_run('funpack') or (warn "Can't find funpack" and $missing_tools = 1); +my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1); +my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1); +#my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# IPP configuration (including nebulous) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up"); + +unless (defined $logfile) { + $logfile = $this_uri; + $logfile =~ s/fits/burn.log/; +} +$ipprc->redirect_output($logfile) if $logfile; + +# Determine the value of a "good" burntool run. +my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); +} + +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); + +my $burntoolStateGood = 999; +foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } +} +if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR); +} +my $outState = -1 * abs($burntoolStateGood); + +print ">>$burntoolStateGood<<\n"; + +# Set up files +my $rawImfile = $this_uri; +my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + +my $outTable = $this_uri; +$outTable =~ s/fits$/burn.tbl/; +my $outTableReal = $ipprc->file_resolve($outTable, 1); + +my $previousTable; +my $previousTableReal; +if ($previous_uri) { + $previousTable = $previous_uri; + $previousTable =~ s/fits$/burn.tbl/; + $previousTableReal = $ipprc->file_resolve($previousTable, 0); +} + +# Set state to processing: +my $burntool_state = 0; + +my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); +if ($status) { + &my_die("failed to update imfile"); +} +$burntool_state = 1; + + +# funpack +my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); +my $tempPixels = $tempfile->filename; + +$status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); +if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); +} + +# burntool +if (($previousTableReal)&&($previousTableReal ne '')) { + $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN); +} +else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); +} +if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); +} +&my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); + +# Replicate files +$status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); +$status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); + +# Set state to finished. +my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState"; +if ($imfile_state ne 'full') { + $command .= " -set_state full "; +} +$status = vsystem ($command, $REALRUN); +if ($status) { + &my_die("failed to update imfile"); +} + +exit 0; + +sub vsystem { + my $command = shift; + my $realrun = shift; + + print "$command\n"; + + my $status = 0; + if ($realrun) { + $status = system ($command); + } + + return $status; +} + +sub my_die { + my $message = shift; + if ($#_ != -1) { + my $exp_id = shift; + my $class_id = shift; + vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1); + } + printf STDERR "$message\n"; + exit 1; +} + + + Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak.pl (revision 30102) @@ -309,5 +309,8 @@ } - $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); + # only destreak cmf file if this is a new run + if ($run_state eq 'new') { + $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); + } } elsif ($stage eq "chip_bg") { Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_defineruns.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_defineruns.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_defineruns.pl (revision 30102) @@ -78,5 +78,5 @@ foreach my $label (@labels) { my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label"; - $command .= " -dry_run" if $no_update; + $command .= " -pretend" if $no_update; $command .= " -limit $stage_limit" if $stage_limit; $command .= " -set_label $label"; Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_revert.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_revert.pl (revision 30102) @@ -191,5 +191,4 @@ $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); - $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); # we use the mask output from the camera stage for input and replace @@ -204,4 +203,10 @@ } + # only revert sources if this is a new run, not one being updated + if ($run_state eq 'new') { + $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id); + $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id); + } + $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id); # This is somewhat kludgey but it works whether the mask is camera mask or chip mask @@ -209,5 +214,4 @@ $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id); - $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id); } elsif ($stage eq "camera") { $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base); Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/nightly_science.pl (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/nightly_science.pl (revision 30102) @@ -0,0 +1,1635 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use DateTime; + +# +# Set up +################################################################################ + +my $missing_tools = 0; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1); +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); +my $moondata = can_run('moondata') or (warn "Can't find moondata" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $db; +my $debug = 0; +my %metadata_out = (); +$metadata_out{nsState} = 'NIGHTLY_SCIENCE'; +my $loghead; +chomp($loghead = `date`); +print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n"; + +# Grab options +my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual); +my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode); +my ( $registration_status, $burntool_status, $observing_status); +my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); +my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old); +my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); +my ( $confirm_stacks, $check_confirm_stacks ); + +GetOptions( + 'help|h' => \$help, + 'date=s' => \$date, # night to make stacks for + 'camera=s' => \$camera, # camera + 'dbname=s' => \$dbname, # Database name + 'logfile=s' => \$logfile, + 'verbose' => \$verbose, + 'debug' => \$debug, + 'test_mode' => \$test_mode, + 'force_stack_count' => \$force_stack_count, + 'force_diff_count' => \$force_diff_count, + 'this_target_only=s' => \$this_target_only, + 'this_filter_only=s' => \$this_filter_only, + 'this_mode_only=s' => \$this_mode_only, + 'registraion_status' => \$registration_status, + 'burntool_status' => \$burntool_status, + 'check_stacks' => \$check_stacks, + 'queue_stacks' => \$queue_stacks, + 'confirm_stacks' => \$confirm_stacks, + 'check_confirm_stacks' => \$check_confirm_stacks, + 'check_diffs' => \$check_diffs, + 'queue_diffs' => \$queue_diffs, + 'check_sweetspot' => \$check_sweetspot, + 'queue_sweetspot' => \$queue_sweetspot, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, + 'check_dqstats' => \$check_dqstats, + 'queue_dqstats' => \$queue_dqstats, + 'clean_old' => \$clean_old, + ) or pod2usage ( 2 ); +pod2usage( -msg => +"USAGE: automate_stacks.pl + Options: + --help This help. + --date YYYY-MM-DD Work on this date (defaults to today GMT). + --camera Default GPC1. + --dbname Default gpc1. + --verbose + --force_stack_count Force the chip/warp counts. + --force_diff_count Force the chip/warp counts. + --this_target_only Process only a single target. + --this_filter_only Process only a single filter. + --this_mode_only Process only a single clean mode. + --burntool_status Display Nexp Nimfile Nburntooled Nqueued for check_chips. + Modes: + --check_stacks Confirm that stacks can be built. + --queue_stacks Issue stacktool commands to queue stacks. + --check_diffs Confirm that diffs can be done. + --queue_diffs Issue difftool commands to queue diffs. + --check_sweetspot See if we should queue SweetSpot stacks. + --queue_sweetspot Issue stacktool commands to queue SweetSpot stacks. + --check_detrends Confirm that detrend verify runs can be built. + --queue_detrends Issue dettool commands to queue detrend verify runs. + --check_dqstats Confirm that dqstats tables can be built. + --queue_dqstats Issue dqstatstool commands to queue dqstat tables. + --clean_old Mark old data 'goto_cleanup'.\n", + -exitval => 2, ) if (defined($help)); +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks", + -exitval => 3, + ) unless + defined $check_registration or defined $define_burntool or defined $queue_burntool or + defined $queue_diffs or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or + defined $check_diffs or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or + defined $test_mode or defined $clean_old or defined $check_mode or + defined $confirm_stacks or defined $burntool_status; + +# Configurable parameters from our config file. +my %science_config = (); +my %clean_config = (); +my %detrend_config = (); +my %eon_config = (); +my %cleanmods_list = (); +my %macro_config = (); +my @unrecoverable_quality = (); +my @filter_list = (); + +# Grab the configuration data. +my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; +my $metadata = $mdcParser->parse(join "", @$stdout_buf); +foreach my $entry (@{ $metadata }) { + if (@{ $entry }{name} eq 'CLEAN_MODES') { + my @mode_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@mode_data) { + if (${ $mentry }{name} eq 'MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'COMMAND') { + $clean_config{$this_mode}{COMMAND} = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'RETENTION_TIME') { + $clean_config{$this_mode}{RETENTION_TIME} = ${ $mentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'FILTERS') { + push @filter_list, ${ $entry }{value}; + } + elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') { + push @unrecoverable_quality, ${ $entry }{value}; + } + elsif (${ $entry }{name} eq 'MACRO_DEFINITIONS') { + my @macro_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@macro_data) { + if (${ $mentry }{name} eq 'PROC_MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'MACRO') { + $macro_config{$this_mode} = ${ $mentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'TARGETS') { + my @target_data = @{ ${ $entry }{value} }; + my $this_target = ''; + foreach my $tentry (@target_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_target = ${ $tentry }{value}; + $science_config{$this_target}{DISTRIBUTION} = $this_target; # Set the default dist_group + } + else { + $science_config{$this_target}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'DETRENDS') { + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + } + else { + $detrend_config{$this_detrend}{ ${ $dentry }{name} } = ${ $dentry }{value}; + } + } + } + elsif (${ $entry }{name} eq 'END_OF_NIGHT') { + my @eon_data = @{ ${ $entry }{value} }; + my $this_eon = ''; + foreach my $tentry (@eon_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_eon = ${ $tentry }{value}; + } + else { + $eon_config{$this_eon}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } + } + } +} + +if (defined($date)) { + my $time; + if ($date =~ / /) { + ($date,$time) = split / /, $date; + } + elsif ($date =~ /T/) { + ($date,$time) = split /T/, $date; + } + my ($year,$month,$day) = split /-/, $date; + if (defined($time)) { + my ($hour,$min,$sec) = split /\:/, $time; #/; + $datetime = DateTime->new( year => $year, + month => $month, + day => $day, + hour => $hour, + min => $min, + second => $sec, + time_zone => 'UTC'); + } + else { + $datetime = DateTime->now(time_zone => 'UTC'); + $datetime->set_year($year); + $datetime->set_month($month); + $datetime->set_day($day); + } +} +else { + $datetime = DateTime->now(time_zone => 'UTC'); # time_zone => 'Pacific/Honolulu'); + $date = $datetime->ymd(); +} +unless(defined($camera)) { + $camera = 'GPC1'; +} +unless(defined($dbname)) { + $dbname = 'gpc1'; +} + +if (defined($this_target_only)) { + foreach my $t (keys %science_config) { + if ($t ne $this_target_only) { + undef($science_config{$t}); + } + } + die("$this_target_only is invalid.") if (scalar keys %science_config < 1); +} + +if (defined($this_filter_only)) { + foreach my $t (@filter_list) { + if ($t eq $this_filter_only) { + @filter_list = ($this_filter_only); + last; + } + } + die("$this_filter_only is invalid.") if ($#filter_list != 0); +} + +if (defined($this_mode_only)) { + foreach my $t (keys %clean_config) { + if ($t ne $this_mode_only) { + undef($clean_config{$t}); + } + } + die("$this_mode_only is invalid.") if (scalar keys %clean_config < 1); +} + +$metadata_out{nsObservingState} = &get_observing_state($date); +# +# Mode selection +################################################################################ + +if (defined($test_mode)) { + $debug = 1; + my $z; + my $v; + foreach $z (sort (keys %science_config)) { + foreach $v (keys %{ $science_config{$z} }) { + print "SCIENCE: $z $v $science_config{$z}{$v}\n"; + } + } + foreach $z (sort (keys %detrend_config)) { + foreach $v (keys %{ $detrend_config{$z} }) { + print "DETREND: $z $v $detrend_config{$z}{$v}\n"; + } + } + foreach $z (keys (%clean_config)) { + foreach $v (keys %{ $clean_config{$z} }) { + print "CLEAN: $z $v $clean_config{$z}{$v}\n"; + } + } + foreach $z (@filter_list) { + print "FILTER: $z\n"; + } + foreach $z (keys (%macro_config)) { + print "MACROS: $z $macro_config{$z}\n"; + } +} +if (defined($observing_status) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECK_OBSERVING_STATUS'; + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($registration_status) || defined($test_mode)) { + &check_summit_copy($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($burntool_status) || defined($test_mode)) { + &burntool_status($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($check_dqstats) || defined($test_mode)) { + &execute_dqstats($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_dqstats)) { + &execute_dqstats($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($check_detrends) || defined($test_mode)) { + $metadata_out{nsDetState} = 'CHECKDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_detrends)) { + $metadata_out{nsDetState} = 'QUEUEDETRENDS'; + &execute_detrends($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} + +if (defined($clean_old) || defined($test_mode)) { + if (defined($test_mode)) { + &execute_cleans($date,"pretend"); + } + else { + &execute_cleans($date); + } + unless (defined($test_mode)) { exit(0); } +} + +if (defined($check_stacks) || defined($test_mode)) { + $metadata_out{nsStackState} = 'TOWARP'; + &execute_stacks($date,$metadata_out{nsObservingState},"pretend"); + if ($metadata_out{nsStackState} eq 'FORCETOWARP') { + $metadata_out{nsStackState} = 'TOWARP'; + } + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_stacks)) { + $metadata_out{nsStackState} = 'STACKING'; + &execute_stacks($date,$metadata_out{nsObservingState}); + if ($metadata_out{nsStackState} eq 'QUEUESTACKING') { + $metadata_out{nsStackState} = 'STACKING_POSSIBLE_ERROR'; + } + return_metadata($date); + exit(0); +} + +if (defined($check_diffs) || defined($test_mode)) { + $metadata_out{nsDiffState} = 'TOWARP'; + &execute_diffs($date,$metadata_out{nsObservingState},"pretend"); + if ($metadata_out{nsDiffState} eq 'FORCETOWARP') { + $metadata_out{nsDiffState} = 'TOWARP'; + } + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($queue_diffs)) { + $metadata_out{nsDiffState} = 'DIFFING'; + &execute_diffs($date,$metadata_out{nsObservingState}); + if ($metadata_out{nsDiffState} eq 'QUEUESTACKING') { + $metadata_out{nsDiffState} = 'STACKING_POSSIBLE_ERROR'; + } + return_metadata($date); + exit(0); +} +exit(10); +if (defined($check_confirm_stacks) || defined($test_mode)) { + $metadata_out{nsStackState} = 'CONFIRM_STACKING'; + &confirm_stacks($date,"pretend"); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($confirm_stacks)) { + $metadata_out{nsStackState} = 'CONFIRM_STACKING'; + &confirm_stacks($date); + return_metadata($date); + exit(0); +} + +if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) { + $metadata_out{nsSSState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_sweetspot)) { + $metadata_out{nsSSState} = 'CHECKSWEETSPOT'; + &execute_sweetspot($date); + return_metadata($date); + exit(0); +} + +exit(0); +# +# Registration +################################################################################ +# This isn't used, but might be useful to keep. + +sub check_summit_copy { + my $date = shift; + my $db = init_gpc_db(); + + # largely stolen from Bill's checkexp program. + + my $sth = " SELECT exp_name, summitExp.dateobs AS registered, rawExp.dateobs, summitExp.imfiles, "; + $sth .= " summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, "; + $sth .= " count(pzDownloadImfile.class_id) AS download_count, newExp.state AS newExp_state, newExp.exp_id, summitExp.exp_type"; + $sth .= " FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) "; + $sth .= " LEFT JOIN newExp ON exp_name = tmp_exp_name LEFT JOIN rawExp USING(exp_id, exp_name) "; + $sth .= " WHERE date(summitExp.dateobs) >= '${date}T00:00:00' AND date(summitExp.dateobs) <= '${date}T23:59:59' "; + $sth .= " GROUP BY exp_name ORDER BY summitExp.dateobs "; + + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nsummit_exps = 0; + my $Nsummit_faults = 0; + my $Ndownload_faults = 0; + my $Nregister_faults = 0; + my $Ndownload_imfiles = 0; + my $Nsummit_imfiles = 0; +# my_trace($sth,$data_ref,$#{ $data_ref }); + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault, + $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref }; + $Nsummit_imfiles += $imfiles; + $Ndownload_imfiles += $download_count; + $Nsummit_exps++; + if ($summit_fault) { + print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nsummit_faults++; + } + } + elsif (!$download_state or $download_state eq 'run') { + print STDERR "check_summit_copy: $date $exp_name has download_state $download_state"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Ndownload_faults++; + } + } + elsif (!$new_state or $new_state eq 'run' ) { + print STDERR "check_summit_copy: $date $exp_name has new_state $new_state"; + if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { + print STDERR " (but I don't care).\n"; + } + else { + print STDERR "\n"; + $Nregister_faults++; + } + } + } + + my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults; + if ($Nsummit_exps == 0) { + print STDERR "No exposures were found on the summit for $date.\n"; + $metadata_out{nsRegState} = 'DROP'; + } + elsif ($Nfaults != 0) { + print STDERR "There were faults while downloading the exposures for $date.\n"; + $metadata_out{nsRegState} = 'NEW'; + } + elsif ($Ndownload_imfiles != $Nsummit_imfiles) { + print STDERR "Not done downloading from the summit for $date (Summit: $Nsummit_imfiles, Downloaded: $Ndownload_imfiles)\n"; + $metadata_out{nsRegState} = 'NEW'; + } + else { + print STDERR "Summit copy and Registration have succeeded for $date.\n"; + $metadata_out{nsRegState} = 'REGISTERED'; + } + + return($Nsummit_exps,$Nfaults); +} + +# +# Burntool +################################################################################ + +sub get_goodBTvalue { + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR); + my $burntoolStateGood = 999; + foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } + } + if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR); + } + return($burntoolStateGood); +} + +sub verify_burntool { + my $exp_id = shift; + my $burntoolStateGood = shift; + + my $db = init_gpc_db(); + + my $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + my $Nimfiles = 0; + my $Nburntooled = 0; + + foreach my $row_ref (@{ $data_ref }) { + my ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref }; + $Nimfiles++; + if (abs($burntool_state) == $burntoolStateGood) { + $Nburntooled++; + } + } + return($Nimfiles,$Nburntooled); +} + +# +# Chips +################################################################################ +# This no longer queues chips, it only checks the counts and returns that. +sub construct_chiptool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $select = "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$chiptool"; + $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp "; + $cmd .= " -set_label $label "; + $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; + $cmd .= " -set_tess_id $tess_id -set_data_group $data_group "; + if (defined($obs_mode)) { + $cmd .= " -obs_mode '$obs_mode' "; + } + if (defined($object)) { + $cmd .= " -object '$object' "; + } + if (defined($comment)) { + $cmd .= " -comment '$comment' "; + } + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= " -pretend "; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_chip { + my $exp_id = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub check_chip_status { + my $date = shift; + my $target = shift; + + my $command = construct_chiptool_cmd($date,$target) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + my $burntoolStateGood = get_goodBTvalue(); + my $Nimfiles = 0; + my $Nburntooled = 0; + my $Nalready = 0; + my @input_exposures = split /\n/, (join '', @$stdout_buf); + foreach my $entry (@input_exposures) { + my ($exp_id, @trash) = split /\s+/, $entry; + @trash = verify_burntool($exp_id,$burntoolStateGood); + $Nimfiles += $trash[0]; + $Nburntooled += $trash[1]; + + @trash = verify_uniqueness_chip($exp_id,$date,$target); + $Nalready += $trash[0]; + } + return($#input_exposures + 1,$Nimfiles,$Nburntooled,$Nalready); +} + +sub burntool_status { + my $date = shift; + my $exposures = 0; + + foreach my $target (sort (keys %science_config)) { + my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = check_chip_status($date,$target); + print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n"; + } +} + + +# +# DQstats +################################################################################ + +sub construct_dqstats_cmd { + my $date = shift; + + my $select = "-dateobs_end ${date}T23:59:59 "; + + my $cmd = "$dqstatstool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " $select "; + $cmd .= " -label %.nightlyscience "; + $cmd .= " -set_label dqstats.nightlyscience "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub pre_dqstats_queue { + my $date = shift; + + my $command = construct_dqstats_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + + return($#input_exposures + 1,1,1); +} + +sub dqstats_queue { + my $date = shift; + + my $command = construct_dqstats_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); + +} +sub execute_dqstats { + my $date = shift; + my $pretend = shift; + my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); + if ($Nexposures == 0) { + print STDERR "execute_dqstats: No exposures on date $date.\n"; + } + elsif ($Ncams != $Nchips) { + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + } + else { + unless(defined($pretend)) { + dqstats_queue($date); + } + if (defined($pretend)) { + add_to_macro_list('check_dqstats',1,$date); + } + else { + add_to_macro_list('queue_dqstats',1,$date); + } + } +} +# +# Detrend verification +################################################################################ + +sub construct_dettool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$dettool"; +# $cmd .= " -pretend "; + $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type "; + $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter "; + $cmd .= " $select "; + $cmd .= " -inst $camera "; + $cmd .= " -select_exp_type $exp_type "; + $cmd .= " -select_filter $filter " if defined($filter); + $cmd .= " -workdir $workdir "; + $cmd .= " -label $label "; + $cmd .= " $use_limits "; + if ($maxN > 0) { + $cmd .= " -random_subset -random_limit $maxN "; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_detverify { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $db = init_gpc_db(); + my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target) . ' -pretend '; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + return($#input_exposures + 1); +} + +sub detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $metadata_out{nsDetState} = 'DETREND_QUEUED'; + return(0); + +} +sub execute_detrends { + my $date = shift; + my $pretend = shift; + my $exposures = 0; + foreach my $target (sort (keys %detrend_config)) { + my ($Nexposures) = pre_detrend_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; + next; + } + $exposures++; + unless(defined($pretend)) { + detrend_queue($date,$target); + } + if (defined($pretend)) { + add_to_macro_list('check_detrends',1,$date,$target,"dettool"); + } + else { + add_to_macro_list('queue_detrends',1,$date,$target,"dettool"); + } + } + if ($exposures == 0) { + $metadata_out{nsDetState} = 'DETREND_DROP'; + } + if (($metadata_out{nsDetState} eq 'CHECKDETRENDS') && ($exposures > 0)) { + $metadata_out{nsDetState} eq 'QUEUEDETRENDS'; + } +} + +# # +# # SweetSpot Stacking +# ################################################################################ +# sub construct_sweetspot_cmd { +# my $date = shift; +# my $target = 'SweetSpot'; +# my $filter = 'w.00000'; + +# my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + +# # Dateobs begin end? +# my ($dateobs_begin,$dateobs_end) = get_lunation_extent($date); +# my $select = "-select_dateobs_begin ${dateobs_begin}T00:00:00 -select_dateobs_end ${dateobs_end}T00:00:00"; +# my $cmd = "$stacktool"; +# $cmd .= " -simple -dbname $dbname -definebyquery "; +# $cmd .= " -set_label SweetSpot.refstack -select_label $label "; +# $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; +# $cmd .= " -select_filter $filter -set_data_group $data_group "; +# $cmd .= " -select_good_frac_min 0.1 -select_fwhm_major_max 8.0 "; +# $cmd .= " -min_num 7 -min_new 4"; +# $cmd .= " $select "; +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# } +# print STDERR "$cmd\n"; +# return($cmd); +# } + +# sub get_lunation_extent { +# my $date = shift; +# my ($year,$month,$day) = split /-/,$date; +# my $dateobs_begin; +# my $dateobs_end; + +# my $dt = DateTime->new(year => $year, month => $month, day => $day, +# hour => 0, minute => 0, second => 0, nanosecond => 0, +# time_zone => 'Pacific/Honolulu'); +# do { +# $dt->subtract(days => 1); +# my $ymd = $dt->ymd; +# my $md_cmd = "moondata $ymd 0 0"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $md_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); +# } +# my @result = split /\s+/,(join "\n", @$stdout_buf); +# if (abs($result[6]) <= 0.5) { +# $dateobs_end = $ymd; +# } +# } while (!defined($dateobs_end)); + +# do { +# $dt->subtract(days => 1); +# my $ymd = $dt->ymd; +# my $md_cmd = "moondata $ymd 0 0"; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $md_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); +# } +# my @result = split /\s+/,(join "", @$stdout_buf); +# if (abs($result[6]) <= 0.5) { +# $dateobs_begin = $ymd; +# } +# } while (!defined($dateobs_begin)); + +# return($dateobs_begin,$dateobs_end); +# } + +# sub pre_sweetspot_queue { +# my $date = shift; +# my $command = construct_sweetspot_cmd($date) . ' -pretend '; +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# my @stacks = split /\n/, (join '', @$stdout_buf); +# my $Nstacks = $#stacks + 1; + +# return(1,$Nstacks); +# } + +# sub sweetspot_queue { +# my $date = shift; +# my $command = construct_sweetspot_cmd($date); +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $command, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# my @stacks = split /\n/, (join '', @$stdout_buf); +# my $Nstacks = $#stacks + 1; + +# return(1,$Nstacks); +# } + +# sub execute_sweetspot { +# my $date = shift; +# my $pretend = shift; + +# my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date); +# if ($Nstacks == 0) { +# print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; +# $metadata_out{nsSSState} = 'SS_EMPTY'; +# return(); +# } +# if ($Nstacks < 10) { +# print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; +# $metadata_out{nsSSState} = 'SS_FEW'; +# return(); +# } +# if ($is_lunation_date == 0) { +# print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; +# $metadata_out{nsSSState} = 'SS_ERROR'; +# return(); +# } +# $metadata_out{nsSSState} = 'SS_QUEUE'; +# $metadata_out{nsSweetSpot} = $Nstacks; +# unless(defined($pretend)) { +# $metadata_out{nsSSState} = 'SS_DONE'; +# sweetspot_queue($date); +# } +# if (defined($pretend)) { +# add_to_macro_list('check_sweetspot',1,$date); +# } +# else { +# add_to_macro_list('queue_sweetspot',1,$date); +# } +# } + +# +# Stacking +################################################################################ + + +sub construct_stacktool_cmd { + my $date = shift; + my $target = shift; + my $filter = shift; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$stacktool"; +# $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " -set_label $label -select_label $label "; + $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; + $cmd .= " -select_filter $filter -set_data_group $data_group "; + $cmd .= " -min_num 4"; + $cmd .= " -select_good_frac_min 0.05"; + $cmd .= " $select "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_stack { + my $skycell = shift; + my $date = shift; + my $target = shift; + my $filter = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_stack_queue { + my $date = shift; + my $observing_state = shift; + my $target = shift; + my $filter = shift; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $where = " label = '$label' AND data_group = '$data_group' "; + + my $where_possibly_faulted = $where . " AND ( " ; + foreach my $acceptable_quality (@unrecoverable_quality) { + $where_possibly_faulted .= " quality = $acceptable_quality OR "; + } + $where_possibly_faulted .= " 0 )"; + + my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; + my $cam_sth = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted "; + my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; + + my $chip_ref = $db->selectall_arrayref( $chip_sth ); + my $cam_ref = $db->selectall_arrayref( $cam_sth ); + my $warp_ref = $db->selectall_arrayref( $warp_sth ); + + # check that we will be able to queue them up + my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR); + } + + my $Nalready = 0; + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + foreach my $entry (@input_exposures) { + my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry; + @trash = verify_uniqueness_stack($skycell,$date,$target,$filter); + $Nalready += $trash[0]; + } + + my $minimum_required_warp; + if ($observing_state eq 'OBSERVING') { + $minimum_required_warp = $science_config{$target}{MIN_STACK}; + } + unless (defined($minimum_required_warp)) { + $minimum_required_warp = 0; + } + + return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $minimum_required_warp, $Nalready); +} + +sub stack_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + + my $command = construct_stacktool_cmd($date,$target,$filter); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + + return(0); +} + +sub execute_stacks { + my $date = shift; + my $observing_state = shift; + my $pretend = shift; + + foreach my $target (sort (keys %science_config)) { + if ($science_config{$target}{STACKABLE} == 1) { + foreach my $filter (@filter_list) { + my ($Nexposures,$NprocChips,$NprocWarps,$NrequiredWarps,$Nalready) = pre_stack_queue($date,$observing_state,$target,$filter); + if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( + if ($debug == 1) { + print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } + $metadata_out{nsStackState} = 'FORCETOWARP'; + next; + } + if ($Nexposures == 0) { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + } + next; + } + if ($NprocWarps < $NrequiredWarps) { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has too few warps to begin stacking. ($NprocWarps $NrequiredWarps)\n"; + } + next; + } + if ($Nalready != 0) { + if ($debug == 1) { + print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + } + unless ($metadata_out{nsStackState} eq 'FORCETOWARP') { + $metadata_out{nsStackState} = 'STACKING'; + } + next; + } + unless (($metadata_out{nsStackState} eq 'FORCETOWARP')||($metadata_out{nsStackState} eq 'STACKING')) { + $metadata_out{nsStackState} = 'QUEUESTACKS'; + } + unless(defined($pretend)) { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } + stack_queue($date,$target,$filter); + } + if (defined($pretend)) { + add_to_macro_list('check_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); + } + else { + add_to_macro_list('queue_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); + } + } + } + else { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target is not auto-stackable.\n"; + } + } + } +} + +sub post_stack_queue { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + # check warp stage == chip stage + my $db = init_gpc_db(); + + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $where = " label = '$label' AND data_group = '$data_group' "; + + my $stack_full_sth = "SELECT * from stackRun where $where "; + my $stack_done_sth = "SELECT * from stackRun LEFT JOIN stackSumSkyfile USING(stack_id) WHERE $where "; + $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) "; + if ($debug == 1) { + print STDERR "post_stack_queue: database queries:\n"; + print STDERR "$stack_full_sth\n"; + print STDERR "$stack_done_sth\n"; + } + my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth ); + my $stack_done_ref = $db->selectall_arrayref( $stack_done_sth ); + + return($#{ $stack_full_ref } + 1, $#{ $stack_done_ref } + 1); +} + +sub confirm_stacks { + my $date = shift; + my $pretend = shift; + + foreach my $target (sort (keys %science_config)) { + if ($science_config{$target}{STACKABLE} == 1) { + my ($Nstacks, $Nattempted) = post_stack_queue($date,$target); + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n"; + } + if ($Nstacks != $Nattempted) { + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"; + } + if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { + $metadata_out{nsStackState} = 'STACKING'; + } + next; + } + if ($Nstacks == 0) { + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n"; + } + next; + } + if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { + if (defined($pretend)) { + add_to_macro_list('check_confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); + } + else { + add_to_macro_list('confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); + } + } + } + } +} + +# +# Extra processing +################################################################################ + +sub verify_uniqueness_diff { + my $warp_id_1 = shift; + my $warp_id_2 = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_diff_queue { + my $date = shift; + my $observing_state = shift; + my $target = shift; + my $filter = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $where = " label = '$label' AND data_group = '$data_group' "; + + my $where_possibly_faulted = $where . " AND ( " ; + foreach my $acceptable_quality (@unrecoverable_quality) { + $where_possibly_faulted .= " quality = $acceptable_quality OR "; + } + $where_possibly_faulted .= " 0 )"; + + my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; + my $cam_sth = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted "; + my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; + + my $chip_ref = $db->selectall_arrayref( $chip_sth ); + my $cam_ref = $db->selectall_arrayref( $cam_sth ); + my $warp_ref = $db->selectall_arrayref( $warp_sth ); + + return($#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2); +} + +sub execute_diffs { + my $date = shift; + my $observing_state = shift; + my $pretend = shift; + + foreach my $target (sort (keys %science_config)) { + if ($science_config{$target}{DIFFABLE} == 1) { + foreach my $filter (@filter_list) { + my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter); + if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) { + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } + $metadata_out{nsDiffState} = 'FORCETOWARP'; + next; + } + if ($NprocChips == 0) { + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n"; + } + next; + } + unless (($metadata_out{nsDiffState} eq 'FORCETOWARP')||($metadata_out{nsDiffState} eq 'DIFFING')) { + $metadata_out{nsDiffState} = 'QUEUEDIFFS'; + } + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } + diff_queue($date,$target,$filter,$pretend); + + if (defined($pretend)) { + add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); + } + else { + add_to_macro_list('queue_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); + } + } + } + else { + if ($debug == 1) { + print STDERR "execute_diffs: Target $target is not auto-diffable.\n"; + } + } + } +} + +sub diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; + + my $object_ref = $db->selectall_arrayref( $obj_sth ); + + foreach my $object_row (@{ $object_ref }) { + my $this_object = shift @{ $object_row }; + my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n"; + next; + } + + while ($#{ $warps } > -1) { + my $input_warp = shift @{ $warps }; + my $template_warp = shift @{ $warps }; + my $input_exp_id = ${ $input_warp }[0]; + my $template_exp_id = ${ $template_warp }[0]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n"; + next; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "diff_queue: $cmd\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + } +} + + + +# +# Auto-Clean +################################################################################ + +sub construct_cleantool_args { + my $date = shift; + my $target = shift; + my $mode = shift; + + my $command = $clean_config{$mode}{COMMAND}; + my $retention_time; + if (exists($science_config{$target}{$mode})) { + $retention_time = $science_config{$target}{$mode}; + } + else { + $retention_time = $clean_config{$mode}{RETENTION_TIME}; + } + if ($retention_time <= 0) { + return("no clean","true"); + } + + my ($year,$month,$day) = split /-/,$date; + my $dt = DateTime->new(year => $year, month => $month, day => $day, + hour => 0, minute => 0, second => 0, nanosecond => 0, + time_zone => 'Pacific/Honolulu'); + + $dt->subtract(days => $retention_time); + my $cleaning_date = $dt->ymd; + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); + + $command =~ s/\@DBNAME\@/$dbname/; + $command =~ s/\@LABEL\@/$label/; + $command =~ s/\@WORKDIR\@/$workdir/; + $command =~ s/\@OBSMODE\@/$obs_mode/; + $command =~ s/\@OBJECT\@/$object/; + $command =~ s/\@TESS_ID\@/$tess_id/; + $command =~ s/\@DIST_GROUP\@/$dist_group/; + $command =~ s/\@DATA_GROUP\@/$data_group/; + $command =~ s/\@REDUCTION\@/$reduction/; + $command =~ s/\@CURRENT_DATE\@/$cleaning_date/; + + if ($debug == 1) { + $command .= ' -pretend '; + } + return($cleaning_date,$command); +} + +sub execute_cleans { + my $date = shift; + my $pretend = shift; + + foreach my $mode (sort (keys %clean_config)) { + foreach my $target (sort (keys %science_config)) { + if (exists($science_config{$target}{NOCLEAN})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + if ($cleaning_date eq 'no clean') { + next; + } + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { +# print STDERR "BEAR IS DRIVING!?\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + add_to_macro_list('clean_old',1,$date,$target,$mode); + } + } + } + return(0); +} + +# +# Utilities +################################################################################ + +sub get_observing_state { + my $date = shift; + my $eon_dt = DateTime->new( year => $datetime->year, + month => $datetime->month, + day => $datetime->day, + hour => 6, + minute => 30, + second => 0, + time_zone => 'Pacific/Honolulu'); + + foreach my $eon (keys %eon_config) { + my $command = "$regtool -processedexp -simple "; + $command .= " -dbname $dbname "; + $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); + $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); + $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); + $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @eon_exposures = split /\n/, (join '', @$stdout_buf); + if ($#eon_exposures >= 0) { + return("END_OF_NIGHT"); + } + } + + if (DateTime->compare($datetime,$eon_dt) < 1) { + return("OBSERVING"); + } + else { + return("END_OF_NIGHT"); + } +} + +sub get_tool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $label = "${target}.nightlyscience"; + my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}"; + my $obs_mode = $science_config{$target}{OBSMODE}; + my $object = $science_config{$target}{OBJECT}; + my $comment = $science_config{$target}{COMMENT}; + my $dist_group = $science_config{$target}{DISTRIBUTION}; + my $data_group = "${target}.${trunc_date}"; + my $tess_id = $science_config{$target}{TESS}; + my $reduction = $science_config{$target}{REDUCTION}; + return($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction); +} + +sub get_dettool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $exp_type = $detrend_config{$target}{EXPTYPE}; + my $det_type = $detrend_config{$target}{DETTYPE}; + my $ref_det_id = $detrend_config{$target}{REF_ID}; + my $ref_iter = $detrend_config{$target}{REF_ITER}; + my $maxN = $detrend_config{$target}{MAX_EXP}; + my $det_filter = $detrend_config{$target}{DET_FILTER}; + my $internal_filter; + if (defined($det_filter)) { + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; + } + else { + $internal_filter = ''; + } + unless (defined($maxN)) { + $maxN = 0; + } + my $lc_type = lc($exp_type); + my $label = "${lc_type}${internal_filter}.$trunc_date"; + my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}"; + return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN); +} + +sub add_to_macro_list { + my $proc_mode = shift; + my $do_or_do_not = shift; + my $date = shift; + my $target = shift; + my $mode = shift; + + if ((defined($macro_config{$proc_mode}))&&($do_or_do_not)) { + unless (defined($metadata_out{N_MACROS})) { + $metadata_out{N_MACROS} = 0; + } + my $N = $metadata_out{N_MACROS}; + $metadata_out{"ns${N}Macro"} = $macro_config{$proc_mode}; + if ($debug == 1) { + print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + if (defined($date)&&(defined($target))) { + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) + = get_tool_parameters($date,$target); + $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/; + $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBSMODE\@/$obs_mode/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/; + $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/; + $metadata_out{"ns${N}Macro"} =~ s/\@TESS_ID\@/$tess_id/; + $metadata_out{"ns${N}Macro"} =~ s/\@DIST_GROUP\@/$dist_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@DATA_GROUP\@/$data_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@REDUCTION\@/$reduction/; + } + if (defined($mode)) { + $metadata_out{"ns${N}Macro"} =~ s/\@EXTRA\@/$mode/; + } + if (defined($date)) { + $metadata_out{"ns${N}Macro"} =~ s/\@DATE\@/$date/; + } + if ($debug == 1) { + print STDERR "DONE WITH A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + $metadata_out{N_MACROS} ++; + } +} + +sub init_gpc_db { + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + my $dbserver = 'ippdb01'; + my $dbuser = 'ippuser'; + my $dbpass = 'ippuser'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} + +sub return_metadata { + my $date = shift; + print STDOUT "autoStack MULTI\n\n"; + print STDOUT "autoStack METADATA\n"; + print STDOUT " date STR $date\n"; + foreach my $k (sort(keys %metadata_out)) { + print STDOUT " $k STR $metadata_out{$k}\n"; + } + print STDOUT "END\n"; +} + +sub my_die { + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + carp($msg); + exit $exit_code; +} + +sub my_trace { + if ($debug == 1) { + foreach my $thing (@_) { + carp($thing); + } + } +} Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/publish_file.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/publish_file.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/publish_file.pl (revision 30102) @@ -44,4 +44,5 @@ my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir ); my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect ); +my ( $output_format ); GetOptions( @@ -59,4 +60,5 @@ 'save-temps' => \$save_temps, # Save temporary files? 'redirect-output' => \$redirect, # Redirect output to log file? + 'output_format=i' => \$output_format, # Output format for ppMops ) or pod2usage( 2 ); @@ -191,4 +193,5 @@ warp_id => $comp->{warp1}, diff_id => $comp->{diff_id}, + output_format => $comp->{output_format}, direction => 1, }; @@ -323,5 +326,4 @@ my $command = "$ppMops $input $output"; - $command .= " -version 1"; # XXX : NOTE: for now, MOPS just wants PS1_DV1 (remove this eventually...) $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name}; $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id}; @@ -335,4 +337,5 @@ $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err}; $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom}; + $command .= " -version " . $data->{output_format} if defined $data->{output_format}; unless ($no_op) { Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/receive_file.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/receive_file.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/receive_file.pl (revision 30102) @@ -32,4 +32,5 @@ my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1); my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1); +my $gunzip = can_run('gunzip') or (warn "Can't find gunzip" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -404,4 +405,24 @@ my $workdir = shift; + my $filecmd_output = `file $src`; + &my_die("failed to determinte file type of $src", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$filecmd_output; + chomp $filecmd_output; + + my $tmpName; + if ($filecmd_output =~ /gzip/) { + my $tmpfile; + ($tmpfile, $tmpName) = tempfile( "/tmp/receive.XXXX", UNLINK => !$save_temps ); + close($tmpfile) or &my_die("failed to close $tmpName", $file_id, $PS_EXIT_UNKNOWN_ERROR); + + my $command = "$gunzip -c $src > $tmpName"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + $src = $tmpName; + } + open my $IN, "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR); open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR); Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl (revision 30102) @@ -30,4 +30,5 @@ my $regtool = can_run('regtool') or (warn "can't find regtool" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); if ($missing_tools) { warn ("Can't find required tools"); @@ -118,4 +119,12 @@ if (uc($exp_type) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +my ($data_group,$dist_group,$chip_workdir,$reduction); +($label,$data_group,$dist_group,$end_stage, + $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"), + &value_for_flag($cmdflags,"-exp_type"), + &value_for_flag($cmdflags,"-obs_mode"), + &value_for_flag($cmdflags,"-object"), + &value_for_flag($cmdflags,"-comment"), + $dbname); my $command = "$regtool -addprocessedexp"; $command .= " -exp_id $exp_id"; @@ -127,4 +136,8 @@ $command .= " -end_stage $end_stage" if defined $end_stage; $command .= " -tess_id $tess_id" if defined $tess_id; +$command .= " -data_group $data_group" if defined $data_group; +$command .= " -dist_group $dist_group" if defined $dist_group; +$command .= " -chip_workdir $chip_workdir" if defined $chip_workdir; +$command .= " -reduction $reduction" if defined $reduction; $command .= " -state full"; $command .= " $cmdflags"; @@ -156,4 +169,145 @@ ### Pau. +sub advance_decisions { + my $dateobs = shift; + my $exptype = shift; + my $obsmode = shift; + my $object = shift; + my $comment = shift; + my $dbname = shift; + + # The strings come pre-quoted. + $dateobs =~ s/\'//g; + $exptype =~ s/\'//g; + $obsmode =~ s/\'//g; + $object =~ s/\'//g; + $comment =~ s/\'//g; + + print "Inside decision engine\n"; + my ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction); + my $target; + + if ($exptype ne 'OBJECT') { + print "Aborting because exptype doesn't claim to be OBJECT: $exptype\n"; + return(undef,undef,undef,"reg",undef,undef,undef); + } + if ($obsmode eq 'ENGINEERING') { + print "Aborting because we don't queue engineering data to chip\n"; + return(undef,undef,undef,"reg",undef,undef,undef); + } + # Grab the configuration data. + my %nightlyscience_config = (); + my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $conf_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); + } + print "data loaded\n"; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + foreach my $entry (@{ $metadata }) { + if (${ $entry }{name} eq 'TARGETS') { + my @target_data = @{ ${ $entry }{value} }; + my $this_target = ''; + foreach my $tentry (@target_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_target = ${ $tentry }{value}; + $nightlyscience_config{$this_target}{DISTRIBUTION} = $this_target; + } + elsif (${ $tentry }{name} eq 'DISTRIBUTION') { + $nightlyscience_config{$this_target}{DISTRIBUTION} = ${ $tentry }{value}; + } + elsif (${ $tentry }{name} eq 'TESS') { + $nightlyscience_config{$this_target}{TESS_ID} = ${ $tentry }{value}; + } + elsif (${ $tentry }{name} eq 'OBSMODE') { + $nightlyscience_config{$this_target}{OBSMODE} = ${ $tentry }{value}; + $nightlyscience_config{$this_target}{OBSMODE} =~ s/%/\.\*/g; + } + elsif (${ $tentry }{name} eq 'OBJECT') { + $nightlyscience_config{$this_target}{OBJECT} = ${ $tentry }{value}; + $nightlyscience_config{$this_target}{OBJECT} =~ s/%/\.\*/g; + } + elsif (${ $tentry }{name} eq 'COMMENT') { + $nightlyscience_config{$this_target}{COMMENT} = ${ $tentry }{value}; + $nightlyscience_config{$this_target}{COMMENT} =~ s/%/\.\*/g; + } + elsif (${ $tentry }{name} eq 'STACKABLE') { + $nightlyscience_config{$this_target}{STACKABLE} = ${ $tentry }{value}; + } + elsif (${ $tentry }{name} eq 'DIFFABLE') { + $nightlyscience_config{$this_target}{DIFFABLE} = ${ $tentry }{value}; + } + elsif (${ $tentry }{name} eq 'REDUCTION') { + $nightlyscience_config{$this_target}{REDUCTION} = ${ $tentry }{value}; + } + } + } + } + print "data parsed\n"; + my $found_target; + foreach $target (keys %nightlyscience_config) { +# print "TEST: $target ($obsmode $object $comment) ($nightlyscience_config{$target}{OBSMODE} $nightlyscience_config{$target}{OBJECT} $nightlyscience_config{$target}{COMMENT}\t"; + my $match = 0; + my $possible = 0; + if (exists($nightlyscience_config{$target}{OBSMODE})) { + $possible++; + if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) { + $match++; + } + } + if (exists($nightlyscience_config{$target}{OBJECT})) { + $possible++; + if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) { + $match++; + } + } + if (exists($nightlyscience_config{$target}{COMMENT})) { + $possible++; + if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) { + $match++; + } + } + print "$possible $match\n"; + if (($possible > 0)&&($match == $possible)) { + $found_target = $target; + last; + } + } + unless ($found_target) { + print "No acceptible target found.\n"; + return(undef,undef,undef,"reg",undef,undef,undef); + } + print "Found this target: $found_target\n"; + print "$dateobs\n"; + $target = $found_target; + + my $date = $dateobs; + $date =~ s/T.*//; + my $workdir_date = $date; + $date =~ s/-//g; + $workdir_date =~ s%-%/%g; + print "$dateobs $date $workdir_date\n"; + $label = "${target}.nightlyscience"; + $data_group = "${target}.${date}"; + $dist_group = $nightlyscience_config{$target}{DISTRIBUTION}; + $end_stage = 'warp'; + $tess_id = $nightlyscience_config{$target}{TESS_ID}; + $chip_workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}/"; + if (exists($nightlyscience_config{$target}{REDUCTION})) { + $reduction = $nightlyscience_config{$target}{REDUCTION}; + } + else { + undef($reduction); + } + + + return ($label,$data_group,$dist_group,$end_stage,$tess_id,$chip_workdir,$reduction); +} + + + sub cache_run Index: /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_imfile.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippScripts/scripts/register_imfile.pl (revision 30102) @@ -19,6 +19,7 @@ use File::Basename qw( basename); use IPC::Run; -use IPC::Cmd 0.36 qw( can_run ); +use IPC::Cmd 0.36 qw( can_run run ); use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Metadata::Config; use Math::Trig; @@ -35,4 +36,5 @@ my $ppStatsFromMetadata = can_run( 'ppStatsFromMetadata' ) or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); my $ppConfigDump = can_run( 'ppConfigDump' ) or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ippApplyBurntoolSingle = can_run( 'ipp_apply_burntool_single.pl' ) or (warn "Can't find ipp_apply_burntool_single" and $missing_tools = 1); my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile); @@ -82,4 +84,5 @@ my $cmdflags; +my $burntoolStateGood; # Run ppStats on the input file { @@ -114,5 +117,5 @@ my $burntoolState = 0; my $isGPC1 = 0; - my $burntoolStateGood = 0; + $burntoolStateGood = 0; foreach my $line (split /\n/, $out1) { if ($line =~ /FPA.BURNTOOL.APPLIED/) { @@ -174,7 +177,8 @@ $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; -$command .= " -data_state full"; +$command .= " -data_state check_burntool"; $command .= " $cmdflags"; +print "$command\n"; # determine solar-system parameters my $longitude = &value_for_flag($cmdflags, 0.0, "-longitude"); @@ -206,4 +210,11 @@ } } + +# This might have a race condition +# $date_end = $dateobs; +# $date_start = $dateobs - 30 minutes ? dateobs_UTC_midnight? +# lock file? +# if exp_type = DARK and date > MIDNIGHT HST { wait } +# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); $now_time = localtime(); @@ -222,4 +233,91 @@ print "skipping command: $command\n"; } + +# We now have an imfile in the database, check if we can burntool it. If not, continue on. + +{ + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $class_id = &value_for_flag($cmdflags, 0.0, "-class_id"); + my $check_date = &value_for_flag($cmdflags, 0.0, "-dateobs"); + $check_date =~ s/T.*$//; + my $exp_name = $tmp_exp_name; + + + my $bt_check_command = "$regtool -checkburntoolimfile "; + $bt_check_command .= " -class_id $class_id "; + $bt_check_command .= " -date $check_date "; + $bt_check_command .= " -valid_burntool $burntoolStateGood "; + $bt_check_command .= " -exp_name $exp_name "; + $bt_check_command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $bt_check_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", + $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR); + } + + my @whole = split /\n/, (join "", @$stdout_buf); + my @single = (); + my $exposures; + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from regtool", + $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless + defined $list; + push @{ $exposures }, @$list; + @single = (); + } + } + + # We only care about the final entry, as that contains the exposure we are. + + my $regtool_update = "$regtool -updateprocessedimfile "; + $regtool_update .= "-dbname $dbname " if defined $dbname; + $regtool_update .= "-exp_id $exp_id -class_id $class_id "; + + my $burntool_data = pop(@{ $exposures }); + if ($burntool_data->{burnable} == 0) { + $regtool_update .= " -burntool_state 0 -set_state pending_burntool "; + } + else { + my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname "; + $apply_command .= " --class_id $class_id --exp_id $exp_id "; + $apply_command .= " --this_uri $burntool_data->{uri} "; + $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri}; + $apply_command .= " --imfile_state $burntool_data->{imfile_state} "; + $apply_command .= " --verbose " if $verbose; + print "$apply_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $apply_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code"); + exit($error_code); + } + } + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + IPC::Cmd::run(command => $regtool_update, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } + } else { + print "skipping command: $command\n"; + } + +} + + $now_time = localtime(); Index: /branches/eam_branches/ipp-20101205/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20101205/ippTasks/Makefile.am (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTasks/Makefile.am (revision 30102) @@ -36,5 +36,5 @@ minidvodb.pro \ minidvodbcopy.pro \ - nightly_stacks.pro \ + nightly_science.pro \ burntool.pro \ lossy_compress.pro \ Index: /branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro =================================================================== --- /branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro (revision 30102) @@ -0,0 +1,710 @@ +## nightly_science.pro : -*- sh -*- + +check.globals + +macro nightly.science.on + ns.initday.on + ns.detrends.off + ns.dqstats.on +# ns.sweetspot.on +# ns.registration.on +# ns.burntool.on +# ns.chips.on + ns.stacks.on + ns.diffs.on +end + +macro nightly.science.off + ns.initday.off + ns.detrends.off + ns.dqstats.off +# ns.sweetspot.off +# ns.registration.off +# ns.burntool.off +# ns.chips.off + ns.stacks.off + ns.diffs.off +end + +macro ns.on + nightly.science.on +end + +macro ns.off + nightly.science.off +end + +macro ns.initday.on + task ns.initday.load + active true + end +end + +macro ns.detrends.on + task ns.detrends.load + active true + end +end + +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + +# macro ns.sweetspot.on +# task ns.sweetspot.load +# active true +# end +# end + +# macro ns.registration.on +# task ns.registration.load +# active true +# end +# end + +# macro ns.burntool.on +# task ns.burntool.load +# active true +# end +# task ns.burntool.run +# active true +# end +# end + +# macro ns.chips.on +# task ns.chips.load +# active true +# end +# task ns.chips.run +# active true +# end +# end + +macro ns.stacks.on + task ns.stacks.load + active false + end + task ns.stacks.run + active true + end + task ns.stacks.confirm + active true + end +end + +macro ns.diffs.on + task ns.diffs.load + active false + end + task ns.diffs.run + active true + end +end + +macro ns.initday.off + task ns.initday.load + active false + end +end + +macro ns.detrends.off + task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load + active false + end +end + +# macro ns.sweetspot.off +# task ns.sweetspot.load +# active false +# end +# end + +# macro ns.registration.off +# task ns.registration.load +# active false +# end +# end + +# macro ns.burntool.off +# task ns.burntool.load +# active false +# end +# task ns.burntool.run +# active false +# end +# end + +# macro ns.chips.off +# task ns.chips.load +# active false +# end +# task ns.chips.run +# active false +# end +# end + +macro ns.stacks.off + task ns.stacks.load + active false + end + task ns.stacks.run + active false + end + task ns.stacks.confirm + active false + end + +end + +macro ns.diffs.off + task ns.diffs.load + active false + end + task ns.diffs.run + active false + end +end + +# $ns_regPAGE = 0 +# $ns_burnPAGE = 0 +# $ns_RburnPAGE = 0 +# $ns_RburnCELL = 0 +# $ns_chipPAGE = 0 +# $ns_RchipPAGE = 0 +$ns_stackPAGE = 0 +$ns_RstackPAGE = 0 +$ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 + +book init nsData +# +# Macros to control the book. +# +macro ns.add.date + book newpage nsData $1 + book setword nsData $1 nsState NEW + book setword nsData $1 nsStackState TOWARP + book setword nsData $1 nsDiffState TOWARP +end + +macro ns.show.dates + book npages nsData -var Npages + for i 0 $Npages + book getpage nsData $i -var date + book getword nsData $date nsState -var state + book getword nsData $date nsStackState -var Sstate + book getword nsData $date nsDiffState -var Dstate + book getword nsData $date nsObservingState -var Ostate + echo $date $state $Sstate $Dstate $Ostate + end +end + +macro ns.del.date + book delpage nsData $1 +end + +macro ns.set.date + book setword nsData $1 nsState $2 +end + + + +# +# Start a new date to work on +# +task ns.initday.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 0:00:00 1:00:00 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.initday.log + stderr $LOGDIR/ns.initday.log + $today = `date -u +%Y-%m-%d` + book newpage nsData $today + book setword nsData $today nsState NEW + book setword nsData $today nsStackState TOWARP + book setword nsData $today nsDiffState TOWARP + book setword nsData $today dbname $DB:0 + +# command echo $today + $run = nightly_science.pl --clean_old --date $today --dbname $DB:0 + add_standard_args $run + command $run + end + + task.exit 0 + # nothing to do here? + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Queue off any possible detrend verification runs +# +task ns.detrends.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + trange 23:00:00 23:59:59 -nmax 1 + npending 1 + + task.exec + stdout $LOGDIR/ns.detrends.log + stdout $LOGDIR/ns.detrends.log + $today = `date -u +%Y-%m-%d` + + $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0 + add_standard_args $run + command $run + end + + task.exit 0 + # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date -u +%Y-%m-%d` + + $run = nightly_science.pl --queue_dqstats --date $today --dbname $DB:0 + add_standard_args $run + command $run + + end + + task.exit 0 + # nothing to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished +# +task ns.stacks.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_stackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book getword nsData $date dbname -var $DBNAME + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.load: " $ns_stackPAGE $date $ns_STATE $Npages + end + + $ns_stackPAGE ++ + if ($ns_stackPAGE >= $Npages) set ns_stackPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + + $run = nightly_science.pl --check_stacks --date $date + add_standard_args $run + command $run + + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make stacks. +# + +task ns.stacks.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_RstackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book getword nsData $date dbname -var $DBNAME + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.run: " $ns_RstackPAGE $date $ns_STATE $Npages + end + + $ns_RstackPAGE ++ + if ($ns_RstackPAGE >= $Npages) set ns_RstackPAGE = 0 + option $date + + if (("$ns_STATE" != "QUEUESTACKS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_stacks --date $date + add_standard_args $run + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 +# We've queued up stacking. We're done with this date, so don't reload the page. + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Confirm that all the stacks that can be built have been built, or at least attempted. +# +task ns.stacks.confirm + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.stacks.log + + book getpage nsData $ns_CstackPAGE -var date + book getword nsData $date nsStackState -var ns_STATE + book getword nsData $date dbname -var $DBNAME + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.stacks.confirm: " $ns_CstackPAGE $date $ns_STATE $Npages + end + + $ns_CstackPAGE ++ + if ($ns_CstackPAGE >= $Npages) set ns_CstackPAGE = 0 + option $date + + if ("$ns_STATE" != "STACKING") break + $run = nightly_science.pl --confirm_stacks --date $date + add_standard_args $run + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + +# +# Check to see if the warps are finished +# +task ns.diffs.load + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.diffs.log + + book getpage nsData $ns_diffPAGE -var date + book getword nsData $date nsDiffState -var ns_STATE + book getword nsData $date dbname -var $DBNAME + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.diffs.load: " $ns_diffPAGE $date $ns_STATE $Npages + end + + $ns_diffPAGE ++ + if ($ns_diffPAGE >= $Npages) set ns_diffPAGE = 0 + option $date + + if ("$ns_STATE" != "TOWARP") break + $run = nightly_science.pl --check_diffs --date $date + add_standard_args $run + command $run + end + + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + +# +# Check to see if the warps are finished and so we can make diffs. +# + +task ns.diffs.run + host local + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 120 + npending 1 + + task.exec + stdout NULL + stderr $LOGDIR/ns.diffs.log + + book getpage nsData $ns_RdiffPAGE -var date + book getword nsData $date nsDiffState -var ns_STATE + book getword nsData $date dbname -var $DBNAME + book npages nsData -var Npages + + if ($VERBOSE > 5) + echo "ns.diffs.run: " $ns_RdiffPAGE $date $ns_STATE $Npages + end + + $ns_RdiffPAGE ++ + if ($ns_RdiffPAGE >= $Npages) set ns_RdiffPAGE = 0 + option $date + + if (("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break + $run = nightly_science.pl --queue_diffs --date $date + add_standard_args $run + command $run + end + # success + task.exit 0 +# convert 'stdout' to book format + book delpage nsData $options:0 +# We've queued up diffing. We're done with this date, so don't reload the page. + ipptool2book stdout nsData -uniq -key date + + book getword nsData $options:0 nsNmacros -var ns_Nmacros + if ("$ns_Nmacros" != "NULL") + for i 0 $ns_Nmacros + sprintf macroName "ns%dMacro" $i + book getword nsData $options:0 $macroName -var macroCmd + $macroCmd + end + end + + if ($VERBOSE > 2) + book listbook nsData + end + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + #operation times out? + task.exit timeout + showcommand timeout + end +end + + + +# # +# # Queue sweetspot runs +# # +# task ns.sweetspot.load +# host local +# periods -poll 3600 +# periods -exec $LOADEXEC +# periods -timeout 30 +# trange 02:00:00 02:59:59 -nmax 1 +# npending 1 + +# task.exec +# stdout $LOGDIR/ns.sweetspot.log +# stderr $LOGDIR/ns.sweetspot.log +# $today = `date -u +%Y-%m-%d` + +# command nightly_science.pl --queue_sweetspot --date $today +# end + +# task.exit 0 +# # nothing to do here +# end +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# # operation times out +# task.exit timeout +# showcommand timeout +# end +# end + Index: /branches/eam_branches/ipp-20101205/ippTasks/nightly_stacks.pro =================================================================== --- /branches/eam_branches/ipp-20101205/ippTasks/nightly_stacks.pro (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTasks/nightly_stacks.pro (revision 30102) @@ -7,9 +7,10 @@ ns.detrends.off ns.dqstats.on - ns.sweetspot.on - ns.registration.on - ns.burntool.on - ns.chips.on +# ns.sweetspot.on +# ns.registration.on +# ns.burntool.on +# ns.chips.on ns.stacks.on + ns.diffs.on end @@ -18,9 +19,10 @@ ns.detrends.off ns.dqstats.off - ns.sweetspot.off - ns.registration.off - ns.burntool.off - ns.chips.off +# ns.sweetspot.off +# ns.registration.off +# ns.burntool.off +# ns.chips.off ns.stacks.off + ns.diffs.off end @@ -51,33 +53,33 @@ end -macro ns.sweetspot.on - task ns.sweetspot.load - active true - end -end - -macro ns.registration.on - task ns.registration.load - active true - end -end - -macro ns.burntool.on - task ns.burntool.load - active true - end - task ns.burntool.run - active true - end -end - -macro ns.chips.on - task ns.chips.load - active true - end - task ns.chips.run - active true - end -end +# macro ns.sweetspot.on +# task ns.sweetspot.load +# active true +# end +# end + +# macro ns.registration.on +# task ns.registration.load +# active true +# end +# end + +# macro ns.burntool.on +# task ns.burntool.load +# active true +# end +# task ns.burntool.run +# active true +# end +# end + +# macro ns.chips.on +# task ns.chips.load +# active true +# end +# task ns.chips.run +# active true +# end +# end macro ns.stacks.on @@ -93,4 +95,13 @@ end +macro ns.diffs.on + task ns.diffs.load + active false + end + task ns.diffs.run + active true + end +end + macro ns.initday.off task ns.initday.load @@ -111,33 +122,33 @@ end -macro ns.sweetspot.off - task ns.sweetspot.load - active false - end -end - -macro ns.registration.off - task ns.registration.load - active false - end -end - -macro ns.burntool.off - task ns.burntool.load - active false - end - task ns.burntool.run - active false - end -end - -macro ns.chips.off - task ns.chips.load - active false - end - task ns.chips.run - active false - end -end +# macro ns.sweetspot.off +# task ns.sweetspot.load +# active false +# end +# end + +# macro ns.registration.off +# task ns.registration.load +# active false +# end +# end + +# macro ns.burntool.off +# task ns.burntool.load +# active false +# end +# task ns.burntool.run +# active false +# end +# end + +# macro ns.chips.off +# task ns.chips.load +# active false +# end +# task ns.chips.run +# active false +# end +# end macro ns.stacks.off @@ -150,16 +161,27 @@ end -$ns_regPAGE = 0 -$ns_burnPAGE = 0 -$ns_RburnPAGE = 0 -$ns_RburnCELL = 0 -$ns_chipPAGE = 0 -$ns_RchipPAGE = 0 +macro ns.diffs.off + task ns.diffs.load + active false + end + task ns.diffs.run + active false + end +end + +# $ns_regPAGE = 0 +# $ns_burnPAGE = 0 +# $ns_RburnPAGE = 0 +# $ns_RburnCELL = 0 +# $ns_chipPAGE = 0 +# $ns_RchipPAGE = 0 $ns_stackPAGE = 0 $ns_RstackPAGE = 0 $ns_CstackPAGE = 0 +$ns_diffPAGE = 0 +$ns_RdiffPAGE = 0 book init nsData -book init nsBurntool +# book init nsBurntool # # Macros to control the book. @@ -177,5 +199,5 @@ echo $date $state end - book npages nsBurntool -var Npages +# book npages nsBurntool -var Npages end @@ -198,5 +220,5 @@ periods -exec $LOADEXEC periods -timeout 30 - trange 16:30:00 17:30:00 -nmax 1 + trange 0:00:00 1:00:00 -nmax 1 npending 1 @@ -204,5 +226,5 @@ stdout $LOGDIR/ns.initday.log stderr $LOGDIR/ns.initday.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` book newpage nsData $today book setword nsData $today nsState NEW @@ -242,5 +264,5 @@ stdout $LOGDIR/ns.detrends.log stdout $LOGDIR/ns.detrends.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_detrends --date $today @@ -278,5 +300,5 @@ stdout $LOGDIR/ns.dqstats.log stderr $LOGDIR/ns.dqstats.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_dqstats --date $today @@ -313,5 +335,5 @@ stdout $LOGDIR/ns.sweetspot.log stderr $LOGDIR/ns.sweetspot.log - $today = `date +%Y-%m-%d` + $today = `date -u +%Y-%m-%d` command automate_stacks.pl --queue_sweetspot --date $today @@ -337,192 +359,192 @@ # Check to see if registration is complete # -task ns.registration.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.registration.log - - book getpage nsData $ns_regPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages - end - - $ns_regPAGE ++ - if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 - option $date - - if ("$ns_STATE" != "NEW") break - $run = automate_stacks.pl --check_registration --date $date - command $run - end - - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Check to see if the chips have been properly burntooled and are ready for queueing -# -task ns.chips.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 300 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.chips.log - - book getpage nsData $ns_chipPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages - end - - $ns_chipPAGE ++ - if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 - option $date - - if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break - $run = automate_stacks.pl --check_chips --date $date - - if ("$ns_STATE" == "BURNING") - $run = $run --isburning - end - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - # remove the burntool page if we're done with it. - book getword nsData $options:0 nsState -var ns_STATE - if ("$ns_STATE" == "QUEUECHIPS") - book delpage nsBurntool $options:0 - end - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Check to see if the chips have been properly burntooled and are ready for queueing -# -task ns.chips.run - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 120 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.chips.log - - book getpage nsData $ns_RchipPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages - end +# task ns.registration.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.registration.log + +# book getpage nsData $ns_regPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.registration.load: " $ns_regPAGE $date $ns_STATE $Npages +# end + +# $ns_regPAGE ++ +# if ($ns_regPAGE >= $Npages) set ns_regPAGE = 0 +# option $date + +# if ("$ns_STATE" != "NEW") break +# $run = automate_stacks.pl --check_registration --date $date +# command $run +# end + +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 300 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_chipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.load: " $ns_chipPAGE $date $ns_STATE $Npages +# end + +# $ns_chipPAGE ++ +# if ($ns_chipPAGE >= $Npages) set ns_chipPAGE = 0 +# option $date + +# if (("$ns_STATE" != "REGISTERED")&&("$ns_STATE" != "BURNING")) break +# $run = automate_stacks.pl --check_chips --date $date + +# if ("$ns_STATE" == "BURNING") +# $run = $run --isburning +# end +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# # remove the burntool page if we're done with it. +# book getword nsData $options:0 nsState -var ns_STATE +# if ("$ns_STATE" == "QUEUECHIPS") +# book delpage nsBurntool $options:0 +# end + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Check to see if the chips have been properly burntooled and are ready for queueing +# # +# task ns.chips.run +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 120 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.chips.log + +# book getpage nsData $ns_RchipPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.chips.run: " $ns_RchipPAGE $date $ns_STATE $Npages +# end - $ns_RchipPAGE ++ - if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 - option $date - - if ("$ns_STATE" != "QUEUECHIPS") break - $BURNTOOLING = 0 - $run = automate_stacks.pl --queue_chips --date $date - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format - book delpage nsData $options:0 - ipptool2book stdout nsData -uniq -key date - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end +# $ns_RchipPAGE ++ +# if ($ns_RchipPAGE >= $Npages) set ns_RchipPAGE = 0 +# option $date + +# if ("$ns_STATE" != "QUEUECHIPS") break +# $BURNTOOLING = 0 +# $run = automate_stacks.pl --queue_chips --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# book delpage nsData $options:0 +# ipptool2book stdout nsData -uniq -key date + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end @@ -557,4 +579,5 @@ command $run end + # success task.exit 0 @@ -718,313 +741,313 @@ end -# -# Generate a list of date ranges and chips that need to be processed with burntool -# -task ns.burntool.load - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - task.exec - stdout NULL - stderr $LOGDIR/ns.burntool.log - - book getpage nsData $ns_burnPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages - end - - $ns_burnPAGE ++ - if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 - option $date +# # +# # Generate a list of date ranges and chips that need to be processed with burntool +# # +# task ns.burntool.load +# host local +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 30 +# npending 1 + +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_burnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.load: " $ns_burnPAGE $date $ns_STATE $Npages +# end + +# $ns_burnPAGE ++ +# if ($ns_burnPAGE >= $Npages) set ns_burnPAGE = 0 +# option $date - if ("$ns_STATE" != "NEEDSBURNING") break - - $run = automate_stacks.pl --define_burntool --date $date - command $run - end - # success - task.exit 0 -# convert 'stdout' to book format -# book delpage nsBurntool $options:0 - ipptool2book stdout nsBurntool -uniq -key date - book setword nsData $options:0 nsState "QUEUEBURNING" - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - showcommand failure - end - task.exit crash - showcommand crash - end - #operation times out? - task.exit timeout - showcommand timeout - end -end - -# -# Magically run burntool on the data, based on the information stored in our book. -# -task ns.burntool.run - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 10800 - npending 30 - task.exec - stdout NULL - stderr $LOGDIR/ns.burntool.log - - book getpage nsData $ns_RburnPAGE -var date - book getword nsData $date nsState -var ns_STATE - book npages nsData -var Npages - - if ($VERBOSE > 5) - echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages - end - - $ns_RburnPAGE ++ - if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 - if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break - $BURNTOOLING = 1 - # Find out where in the list of jobs we are - book getword nsBurntool $date btN -var btN - book getword nsBurntool $date btNCounter -var btNcounter - - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $date - end - - if ("$ns_STATE" == "QUEUEBURNING") - $new_state = "QUEUEBURNING" - end - if ("$ns_STATE" == "BURNING") - $new_state = "BURNING" - end - if ($btNcounter + 1 > $btN) - $new_state = "BURNING" - end - if ($btNcounter > $btN) - $btNcounter = 0 - end - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $new_state - end +# if ("$ns_STATE" != "NEEDSBURNING") break + +# $run = automate_stacks.pl --define_burntool --date $date +# command $run +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # book delpage nsBurntool $options:0 +# ipptool2book stdout nsBurntool -uniq -key date +# book setword nsData $options:0 nsState "QUEUEBURNING" + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# showcommand failure +# end +# task.exit crash +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# showcommand timeout +# end +# end + +# # +# # Magically run burntool on the data, based on the information stored in our book. +# # +# task ns.burntool.run +# periods -poll $LOADPOLL +# periods -exec $LOADEXEC +# periods -timeout 10800 +# npending 30 +# task.exec +# stdout NULL +# stderr $LOGDIR/ns.burntool.log + +# book getpage nsData $ns_RburnPAGE -var date +# book getword nsData $date nsState -var ns_STATE +# book npages nsData -var Npages + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: " $ns_RburnPAGE $date $ns_STATE $Npages +# end + +# $ns_RburnPAGE ++ +# if ($ns_RburnPAGE >= $Npages) set ns_RburnPAGE = 0 +# if (("$ns_STATE" != "QUEUEBURNING")&&("$ns_STATE" != "BURNING")) break +# $BURNTOOLING = 1 +# # Find out where in the list of jobs we are +# book getword nsBurntool $date btN -var btN +# book getword nsBurntool $date btNCounter -var btNcounter + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $date +# end + +# if ("$ns_STATE" == "QUEUEBURNING") +# $new_state = "QUEUEBURNING" +# end +# if ("$ns_STATE" == "BURNING") +# $new_state = "BURNING" +# end +# if ($btNcounter + 1 > $btN) +# $new_state = "BURNING" +# end +# if ($btNcounter > $btN) +# $btNcounter = 0 +# end +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $new_state +# end - # Increment the counter in the book for the next job. - $counter_update = $btNcounter + 1 - book setword nsBurntool $date btNCounter $counter_update -# book setword nsData $options:0 nsState $options:1 - book setword nsData $date nsState $new_state - - # Get the current status of this job, and skip if it doesn't need to process. - sprintf status_label "bt%dStatus" $btNcounter - book getword nsBurntool $date $status_label -var status - - if ($VERBOSE > 5) - echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label - end - if (("$status" == "FINISHED")||("$status" == "RUN")) break +# # Increment the counter in the book for the next job. +# $counter_update = $btNcounter + 1 +# book setword nsBurntool $date btNCounter $counter_update +# # book setword nsData $options:0 nsState $options:1 +# book setword nsData $date nsState $new_state + +# # Get the current status of this job, and skip if it doesn't need to process. +# sprintf status_label "bt%dStatus" $btNcounter +# book getword nsBurntool $date $status_label -var status + +# if ($VERBOSE > 5) +# echo "ns.burntool.run: Status: " $btNcounter $status $date $status_label +# end +# if (("$status" == "FINISHED")||("$status" == "RUN")) break - # Continue loading information to process this job - sprintf start_date_label "bt%dBegin" $btNcounter - sprintf end_date_label "bt%dEnd" $btNcounter - sprintf class_count_label "bt%dClass" $btNcounter - - book getword nsBurntool $date $start_date_label -var start_date - book getword nsBurntool $date $end_date_label -var end_date - book getword nsBurntool $date $class_count_label -var chip_counter - - # Lookup class_id/host pairs - list word -split $hostmatch:$chip_counter - $class_id = $word:0 - $host = $word:1 - host $host -# set.host.for.camera GPC1 $class_id - $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" - - $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile +# # Continue loading information to process this job +# sprintf start_date_label "bt%dBegin" $btNcounter +# sprintf end_date_label "bt%dEnd" $btNcounter +# sprintf class_count_label "bt%dClass" $btNcounter + +# book getword nsBurntool $date $start_date_label -var start_date +# book getword nsBurntool $date $end_date_label -var end_date +# book getword nsBurntool $date $class_count_label -var chip_counter + +# # Lookup class_id/host pairs +# list word -split $hostmatch:$chip_counter +# $class_id = $word:0 +# $host = $word:1 +# host $host +# # set.host.for.camera GPC1 $class_id +# $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log" + +# $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile - echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run - echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state - - book setword nsBurntool $date $status_label RUN - option $date $new_state $status_label - - command $run -# command /bin/true - end - # success - task.exit 0 -# convert 'stdout' to book format - # Set data state based on if we're queueing or waiting - # Set the job state for success. - book setword nsBurntool $options:0 $options:2 FINISHED - - book getword nsData $options:0 nsNmacros -var ns_Nmacros - if ("$ns_Nmacros" != "NULL") - for i 0 $ns_Nmacros - sprintf macroName "ns%dMacro" $i - book getword nsData $options:0 $macroName -var macroCmd - $macroCmd - end - end - - if ($VERBOSE > 2) - book listbook nsData - end - end - - # locked list - task.exit default - book setword nsBurntool $options:0 $options:2 FAIL - showcommand failure - end - task.exit crash - book setword nsBurntool $options:0 $options:2 FAIL - showcommand crash - end - #operation times out? - task.exit timeout - book setword nsBurntool $options:0 $options:2 FAIL - showcommand timeout - end -end - - -# -# This is all burntool related stuff. -# - -macro burntool - if ($0 != 3) - echo "USAGE: burntool (dateobs_begin) (dateobs_end)" - break - end - - for i 0 $hostmatch:n - job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile - end -end -macro loadhosts - for i 0 $allhosts:n - controller host add $allhosts:$i - end -end - -# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. - -list hostmatch - XY01 ipp014 - XY03 ipp038 - XY05 ipp023 - XY10 ipp039 - XY12 ipp024 - XY15 ipp040 - XY17 ipp020 - XY21 ipp041 - XY23 ipp042 - XY25 ipp043 - XY27 ipp028 - XY31 ipp044 - XY33 ipp029 - XY35 ipp045 - XY37 ipp030 - XY41 ipp046 - XY43 ipp031 - XY45 ipp047 - XY47 ipp032 - XY51 ipp048 - XY53 ipp033 - XY55 ipp049 - XY57 ipp034 - XY61 ipp050 - XY63 ipp035 - XY65 ipp051 - XY67 ipp036 - XY72 ipp052 - XY74 ipp015 - XY76 ipp053 - XY02 ipp014 - XY04 ipp038 - XY06 ipp023 - XY11 ipp039 - XY13 ipp024 - XY14 ipp040 - XY16 ipp020 - XY20 ipp041 - XY22 ipp042 - XY24 ipp043 - XY26 ipp028 - XY30 ipp044 - XY32 ipp029 - XY34 ipp045 - XY36 ipp030 - XY40 ipp046 - XY42 ipp031 - XY44 ipp047 - XY46 ipp032 - XY50 ipp048 - XY52 ipp033 - XY54 ipp049 - XY56 ipp034 - XY60 ipp050 - XY62 ipp035 - XY64 ipp051 - XY66 ipp036 - XY71 ipp052 - XY73 ipp015 - XY75 ipp053 -end - -list allhosts - ipp043 - ipp014 - ipp015 - ipp023 - ipp024 - ipp020 - ipp028 - ipp029 - ipp030 - ipp031 - ipp032 - ipp033 - ipp034 - ipp035 - ipp036 - ipp038 - ipp039 - ipp040 - ipp041 - ipp042 - ipp044 - ipp045 - ipp046 - ipp047 - ipp048 - ipp049 - ipp050 - ipp051 - ipp052 - ipp053 -end +# echo "ns.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run +# echo "ns.burntool.run: " $date $btN $btNcounter $chip_counter $new_state + +# book setword nsBurntool $date $status_label RUN +# option $date $new_state $status_label + +# command $run +# # command /bin/true +# end +# # success +# task.exit 0 +# # convert 'stdout' to book format +# # Set data state based on if we're queueing or waiting +# # Set the job state for success. +# book setword nsBurntool $options:0 $options:2 FINISHED + +# book getword nsData $options:0 nsNmacros -var ns_Nmacros +# if ("$ns_Nmacros" != "NULL") +# for i 0 $ns_Nmacros +# sprintf macroName "ns%dMacro" $i +# book getword nsData $options:0 $macroName -var macroCmd +# $macroCmd +# end +# end + +# if ($VERBOSE > 2) +# book listbook nsData +# end +# end + +# # locked list +# task.exit default +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand failure +# end +# task.exit crash +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand crash +# end +# #operation times out? +# task.exit timeout +# book setword nsBurntool $options:0 $options:2 FAIL +# showcommand timeout +# end +# end + + +# # +# # This is all burntool related stuff. +# # + +# macro burntool +# if ($0 != 3) +# echo "USAGE: burntool (dateobs_begin) (dateobs_end)" +# break +# end + +# for i 0 $hostmatch:n +# job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile +# end +# end +# macro loadhosts +# for i 0 $allhosts:n +# controller host add $allhosts:$i +# end +# end + +# # sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time. + +# list hostmatch +# XY01 ipp014 +# XY03 ipp038 +# XY05 ipp023 +# XY10 ipp039 +# XY12 ipp024 +# XY15 ipp040 +# XY17 ipp020 +# XY21 ipp041 +# XY23 ipp042 +# XY25 ipp043 +# XY27 ipp028 +# XY31 ipp044 +# XY33 ipp029 +# XY35 ipp045 +# XY37 ipp030 +# XY41 ipp046 +# XY43 ipp031 +# XY45 ipp047 +# XY47 ipp032 +# XY51 ipp048 +# XY53 ipp033 +# XY55 ipp049 +# XY57 ipp034 +# XY61 ipp050 +# XY63 ipp035 +# XY65 ipp051 +# XY67 ipp036 +# XY72 ipp052 +# XY74 ipp015 +# XY76 ipp053 +# XY02 ipp014 +# XY04 ipp038 +# XY06 ipp023 +# XY11 ipp039 +# XY13 ipp024 +# XY14 ipp040 +# XY16 ipp020 +# XY20 ipp041 +# XY22 ipp042 +# XY24 ipp043 +# XY26 ipp028 +# XY30 ipp044 +# XY32 ipp029 +# XY34 ipp045 +# XY36 ipp030 +# XY40 ipp046 +# XY42 ipp031 +# XY44 ipp047 +# XY46 ipp032 +# XY50 ipp048 +# XY52 ipp033 +# XY54 ipp049 +# XY56 ipp034 +# XY60 ipp050 +# XY62 ipp035 +# XY64 ipp051 +# XY66 ipp036 +# XY71 ipp052 +# XY73 ipp015 +# XY75 ipp053 +# end + +# list allhosts +# ipp043 +# ipp014 +# ipp015 +# ipp023 +# ipp024 +# ipp020 +# ipp028 +# ipp029 +# ipp030 +# ipp031 +# ipp032 +# ipp033 +# ipp034 +# ipp035 +# ipp036 +# ipp038 +# ipp039 +# ipp040 +# ipp041 +# ipp042 +# ipp044 +# ipp045 +# ipp046 +# ipp047 +# ipp048 +# ipp049 +# ipp050 +# ipp051 +# ipp052 +# ipp053 +# end Index: /branches/eam_branches/ipp-20101205/ippTasks/publish.pro =================================================================== --- /branches/eam_branches/ipp-20101205/ippTasks/publish.pro (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTasks/publish.pro (revision 30102) @@ -130,4 +130,5 @@ book getword publishRun $pageName stage_id -var STAGE_ID book getword publishRun $pageName dbname -var DBNAME + book getword publishRun $pageName output_format -var OUTPUT_FORMAT stdout $LOGDIR/publish.run.log @@ -137,5 +138,5 @@ strsub $WORKDIR_TEMPLATE @HOST@ $default_host -var WORKDIR - $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --redirect-output + $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --output_format $OUTPUT_FORMAT --redirect-output add_standard_args run Index: /branches/eam_branches/ipp-20101205/ippTasks/register.pro =================================================================== --- /branches/eam_branches/ipp-20101205/ippTasks/register.pro (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTasks/register.pro (revision 30102) @@ -8,8 +8,10 @@ book init regPendingImfile book init regPendingExp +book init regPendingBurntoolImfile macro register.reset book init regPendingImfile book init regPendingExp + book init regPendingBurntoolImfile end @@ -18,4 +20,5 @@ book listbook regPendingImfile book listbook regPendingExp + book listbook regPendingBurntoolImfile end @@ -33,4 +36,10 @@ active true end + task register.burntool.load + active true + end + task register.burntool.run + active true + end end @@ -48,5 +57,13 @@ active false end -end + task register.burntool.load + active false + end + task register.burntool.run + active false + end +end + +$valid_burntool_value = 14 macro replace.survey.values @@ -129,4 +146,5 @@ # these variables will cycle through the known database names $regPendingImfile_DB = 0 +$regPendingBurntoolImfile_DB = 0 $regPendingExp_DB = 0 @@ -429,3 +447,133 @@ end + +# select imfiles that can now be burntooled. +task register.burntool.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.burntool.load.log + + # select entried from the current DB; cycle to the next DB, if it exists + task.exec + $run = regtool -pendingburntoolimfile + $today = `date -u +%Y-%m-%d` +# debugging purposes +# $today = "2010-12-06" + $run = $run -date $today -valid_burntool $valid_burntool_value + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingBurntoolImfile_DB + $run = $run -dbname $DB:$regPendingBurntoolImfile_DB + $regPendingBurntoolImfile_DB ++ + if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0 + end + +# echo $run + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingBurntoolImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingBurntoolImfile + if ($VERBOSE > 2) + book listbook regPendingBurntoolImfile + end + + # delete existing entries in the appropriate pantasksStates + process_cleanup regPendingBurntoolImfile + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the ipp_apply_burntool_single.pl script on the pending images +task register.burntool.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages regPendingBurntoolImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images to burn in regPendingBurntoolImfile + book getpage regPendingBurntoolImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingBurntoolImfile $pageName pantaskState RUN + + book getword regPendingBurntoolImfile $pageName exp_id -var EXP_ID + book getword regPendingBurntoolImfile $pageName class_id -var CLASS_ID + book getword regPendingBurntoolImfile $pageName uri -var THIS_URI + book getword regPendingBurntoolImfile $pageName previous_uri -var PREVIOUS_URI + book getword regPendingBurntoolImfile $pageName camera -var CAMERA + book getword regPendingBurntoolImfile $pageName dbname -var DBNAME + set.host.for.camera $CAMERA $CLASS_ID + + stdout $LOGDIR/register.burntool.run.log + stderr $LOGDIR/register.burntool.run.log + + $run = ipp_apply_burntool_single.pl --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI + if ("$PREVIOUS_URI" != "NULL") + $run = $run --previous_uri $PREVIOUS_URI + end + + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingBurntoolImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword regPendingBurntoolImfile $options:0 pantaskState CRASH + end + + # operation timed out + task.exit timeout + showcommand timeout + book setword regPendingBurntoolImfile $options:0 pantaskState TIMEOUT + end +end + + + + ## XXX add a global path to output files Index: /branches/eam_branches/ipp-20101205/ippToPsps/docs/loadingSummary.txt =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/docs/loadingSummary.txt (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippToPsps/docs/loadingSummary.txt (revision 30102) @@ -0,0 +1,21 @@ +*********************************************************** +* Summary of IPP->PSPS loading as of 2010-12-09 12:05:49 +* DVO Db = 'ThreePi.V1' +* Exposures between 105437 and 213141 +* +* total Exposures = 35973 +* merged into PSPS = 32082 +* un-merged = 3890 +* unprocessed by ippToPsps = 3351 +* missing SMF files = 518 +* available SMF files = 2833 +* processed by ippToPsps = 539 +* not on datastore = 4 +* on datastore = 535 +* not loaded to ODM = 19 +* loaded to ODM but rejected = 516 +* under -30 dec limit = 493 +* remaining ODM failures = 23 +* +*********************************************************** + Index: /branches/eam_branches/ipp-20101205/ippToPsps/perl/checkOdmStatus.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/perl/checkOdmStatus.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippToPsps/perl/checkOdmStatus.pl (revision 30102) @@ -39,4 +39,7 @@ ); +if (@ARGV) { + print "* UNKNKOWN: option @ARGV\n"; +} if (!defined $product) { print "* OPTIONAL: a datastore product name -p \n"; @@ -159,7 +162,6 @@ $numBatchesToCheck++; - # if not merged then update by polling ODM for status + # if not merged or failed load, then update by polling ODM for status if (!$merged && !$loadFailed) { - if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) { Index: /branches/eam_branches/ipp-20101205/ippToPsps/perl/exposureSummary.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/perl/exposureSummary.pl (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippToPsps/perl/exposureSummary.pl (revision 30102) @@ -0,0 +1,213 @@ +#!/usr/bin/env perl + +# script to provide a summary of loading status given an upper and lower exposure ID limit + +use warnings; +use strict; + +use LWP::UserAgent; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use File::Temp qw(tempfile); +use ippToPsps::IppToPspsDb; +use ippToPsps::Gpc1Db; +use ippToPsps::DetectionBatch; + +my $dvoDb = undef; +my $beginExp = undef; +my $endExp = undef; +my $verbose = undef; +my $checkNeb = undef; +my $save_temps = undef; + +GetOptions( + 'startexp|b=s' => \$beginExp, + 'endexpi|e=s' => \$endExp, + 'dvodb|d=s' => \$dvoDb, + 'checkNeb|n' => \$checkNeb, + 'verbose|v' => \$verbose, + 'save_temps|s' => \$save_temps + ); + + +my $quit = 0; +print "\n*******************************************************************************\n"; +print "* \n"; +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} +if (!defined $beginExp) { + $quit = 1; + print "* REQUIRED: a begin exp_id -b \n"; +} +if (!defined $endExp) { + $quit = 1; + print "* REQUIRED: an end exp_id -e \n"; +} +if (!defined $dvoDb) { + $quit = 1; + print "* REQUIRED: a dvo Db name -d \n"; +} +if (!defined $checkNeb) { + $checkNeb = 0; + print "* OPTIONAL: check nebulous for unprocessed exposures -n (default = $checkNeb)\n"; +} +if (!defined $verbose) { + $verbose = 0; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; +} +if (!defined $save_temps) { + $save_temps = 0; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; +} +print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + +my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); +my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); + +my $exposures; +#if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures, 0)) {exit;} + +my $totalExposures = $ippToPspsDb->countExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); +my $totalMerged = $ippToPspsDb->countMergedExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); + +if(!$ippToPspsDb->getUnmergedExposures(\$exposures, $beginExp, $endExp, "PSPS_test", "P2", $dvoDb)) { + + print "No unmerged exposures\n"; + exit; +} + +my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); +my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; + +my $exposure; +my $totalUnmerged = 0; +my $totalMergeWorthy = 0; +my $totalProcessed = 0; +my $totalOnDatastore = 0; +my $totalLoadedToOdm = 0; +my $totalLoadFailed = 0; +my $totalUnderObjIdLimit = 0; +my $totalUnprocessed = 0; +my $totalUnprocessedAvailableSmfs = 0; +my $totalOdmFailures = 0; +my $totalUnprocessedMissingSmfs = 0; + + +open (UNPRO, ">unprocessedExposures.txt") or print "* Problem opening data file for unprocessed exposures\n"; +open (ODMFAIL, ">odmFailures.txt") or print "* Problem opening data file for ODM failures\n"; + +if ($checkNeb) { + + open (MISSSMF, ">missingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; + open (NOTMISSSMF, ">notMissingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; +} + +foreach $exposure ( @{$exposures} ) { + my ($batchId, $expId) = @{$exposure}; + + if ($ippToPspsDb->isExposureMerged($expId)) {next;} + + $totalUnmerged++; + if ($verbose) {print "$totalUnmerged | $expId \n"; } + + if ($ippToPspsDb->isExposureProcessed($expId)) { + + $totalProcessed++; + + if ($ippToPspsDb->isExposureAlreadyPublished($expId)) { + + $totalOnDatastore++; + + if ($ippToPspsDb->isExposureLoadedToOdm($expId)) { + + $totalLoadedToOdm++; + + if ($ippToPspsDb->didLoadFail($expId)) { + + $totalLoadFailed++; + } + if ($ippToPspsDb->isMinObjIdUnderLimit($expId)) { + + $totalUnderObjIdLimit++; + } + else { + + $totalOdmFailures++; + print ODMFAIL "$expId\n"; + } + } + } + } + else { + + + $totalUnprocessed++; + print UNPRO "$expId\n"; + + # if flag is set, then attempt to get smf file from nebulous for this unprocessed exposure + if ($checkNeb) { + + my $detectionBatch = ippToPsps::DetectionBatch->existing( + "GPC1", + $gpc1Db, + $batchId, + $ippToPspsDb, + ".", + $verbose, + $save_temps); + + + my $path = $detectionBatch->getSmfFile(); + if (!$path) { + print "no smf file for $expId\n"; + + $totalUnprocessedMissingSmfs++; + print MISSSMF "$expId\n"; + } + else { + + $totalUnprocessedAvailableSmfs++; + print NOTMISSSMF "$expId\n"; + + } + } + + } +} + +close(UNPRO); +close(ODMFAIL); +if ($checkNeb) { + + close(MISSSMF); + close(NOTMISSSMF); +} +my $totalProcessedNotOnDatastore = $totalProcessed - $totalOnDatastore; +my $totalOnDatastoreNotLoaded = $totalOnDatastore - $totalLoadedToOdm; +print "***********************************************************\n"; +print "* Summary of IPP->PSPS loading as of $timeStamp\n"; +print "* DVO Db = '$dvoDb'\n"; +print "* Exposures between $beginExp and $endExp\n"; +print "*\n"; +print "* total Exposures = $totalExposures\n"; +print "* merged into PSPS = $totalMerged\n"; +print "* un-merged = $totalUnmerged\n"; +print "* unprocessed by ippToPsps = $totalUnprocessed\n"; +if ($checkNeb) { + print "* missing SMF files = $totalUnprocessedMissingSmfs\n"; + print "* available SMF files = $totalUnprocessedAvailableSmfs\n"; +} +print "* processed by ippToPsps = $totalProcessed\n"; +print "* not on datastore = $totalProcessedNotOnDatastore\n"; +print "* on datastore = $totalOnDatastore\n"; +print "* not loaded to ODM = $totalOnDatastoreNotLoaded\n"; +print "* loaded to ODM but rejected = $totalLoadedToOdm\n"; +print "* under -30 dec limit = $totalUnderObjIdLimit\n"; +print "* remaining ODM failures = $totalOdmFailures\n"; +print "*\n"; +print "***********************************************************\n"; + + Index: /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/DetectionBatch.pm =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/DetectionBatch.pm (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/DetectionBatch.pm (revision 30102) @@ -78,5 +78,5 @@ # get neb path of smf file my $nebPath = $self->{_gpc1Db}->getCameraStageSmfForThisDvoDb($self->{_dvoDb}, $self->{_expId}); - if (!$nebPath) { return 0; } + if (!$nebPath) { return undef; } # get real filename from neb 'key' Index: /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/IppToPspsDb.pm =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 30102) @@ -11,5 +11,5 @@ ########################################################################### # -# Returns a list of batches that have been processed and loaded to datastore +# Returns a list of batches created within the provided dates # ########################################################################### @@ -39,7 +39,5 @@ $query->execute; ${$batches} = $query->fetchall_arrayref(); - my $count = scalar @{${$batches}}; - - return $count; + return scalar @{${$batches}}; } @@ -72,7 +70,75 @@ $query->execute; ${$batches} = $query->fetchall_arrayref(); - my $count = scalar @{${$batches}}; - - return $count; + return scalar @{${$batches}}; +} + +########################################################################### +# +# Returns a count of all distinct exposures between provided limits +# +########################################################################### +sub countExposures { + my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' +SQL + + $query->execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Returns a count of exposures (between provided limits) that have been merged +# +########################################################################### +sub countMergedExposures { + my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' + AND merged +SQL + + $query->execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Returns a list of exposures (between provided limits) that have not been merged +# +########################################################################### +sub getUnmergedExposures { + my ($self, $exposures, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' + AND !merged + GROUP BY exp_id +SQL + + $query->execute; + ${$exposures} = $query->fetchall_arrayref(); + return scalar @{${$exposures}}; } @@ -166,4 +232,103 @@ ####################################################################################### # +# Checks whether this exposure failed to load to the ODM +# +######################################################################################## +sub didLoadFail { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has a min object ID under old PSPS hard limit of 72010000000000001 +# +######################################################################################## +sub isMinObjIdUnderLimit { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} +####################################################################################### +# +# Checks whether this exposure has been processed +# +######################################################################################## +sub isExposureProcessed { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has been merged +# +######################################################################################## +sub isExposureMerged { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has been loaded to ODM +# +######################################################################################## +sub isExposureLoadedToOdm { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# # Checks whether we have successfully processed this exposure and loaded it to the datastore # @@ -184,4 +349,5 @@ my $processed = $query->fetchrow_array(); + # TODO can use return scalar $query->fetchrow_array(); return $processed; } Index: /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps_run.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippToPsps/perl/ippToPsps_run.pl (revision 30102) @@ -15,4 +15,5 @@ use ippToPsps::IppToPspsDb; use ippToPsps::Datastore; +#use ippToPsps::BatchManager; # globals @@ -112,4 +113,5 @@ my $datastore = undef; if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} +#my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $output, $verbose, $save_temps); # check we can run programs and get camera config Index: /branches/eam_branches/ipp-20101205/ippTools/configure.ac =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/configure.ac (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/configure.ac (revision 30102) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.65], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.66], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.65]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.66]) PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) Index: /branches/eam_branches/ipp-20101205/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/Makefile.am (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/Makefile.am (revision 30102) @@ -316,4 +316,5 @@ receivetool_revert.sql \ receivetool_toadvance.sql \ + regtool_checkburntoolimfile.sql \ regtool_create_dup_table.sql \ regtool_export_exp.sql \ @@ -331,4 +332,5 @@ regtool_updatebyquery.sql \ regtool_updatebyqueryimfile.sql \ + regtool_pendingburntoolimfile.sql \ stacktool_associationdefine_select.sql \ stacktool_definebyquery_insert.sql \ Index: /branches/eam_branches/ipp-20101205/ippTools/share/chiptool_find_rawexp.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/chiptool_find_rawexp.sql (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/chiptool_find_rawexp.sql (revision 30102) @@ -1,9 +1,10 @@ -- this query is used to find potental rawExps to be queued for chiptool -- processeing -SELECT +SELECT DISTINCT rawExp.*, newExp.label FROM rawExp JOIN newExp using (exp_id) +LEFT JOIN chipRun USING (exp_id) WHERE rawExp.fault = 0 Index: /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_definerun.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_definerun.sql (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_definerun.sql (revision 30102) @@ -3,5 +3,6 @@ client_id, stage_id, - src_label + src_label, + output_format FROM ( -- Get diffs to publish @@ -9,5 +10,6 @@ client_id, diff_id AS stage_id, - diffRun.label AS src_label + diffRun.label AS src_label, + output_format FROM publishClient JOIN diffRun @@ -28,5 +30,6 @@ client_id, cam_id AS stage_id, - camRun.label AS src_label + camRun.label AS src_label, + output_format FROM publishClient JOIN camRun @@ -43,5 +46,6 @@ client_id, diff_phot_id AS stage_id, - diffPhotRun.label AS src_label + diffPhotRun.label AS src_label, + output_format FROM publishClient JOIN diffPhotRun Index: /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_pending.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_pending.sql (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/pubtool_pending.sql (revision 30102) @@ -9,4 +9,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, diffRun.diff_id AS stage_id, rawExp.camera, @@ -35,4 +36,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, camRun.cam_id AS stage_id, rawExp.camera, @@ -56,4 +58,5 @@ publishClient.stage, publishClient.workdir, + publishClient.output_format, diffPhotRun.diff_phot_id AS stage_id, rawExp.camera, Index: /branches/eam_branches/ipp-20101205/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/pxadmin_create_tables.sql (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/pxadmin_create_tables.sql (revision 30102) @@ -9,4 +9,5 @@ uri VARCHAR(255), epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + use_compress SMALLINT, PRIMARY KEY(camera, telescope) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -251,4 +252,5 @@ md5sum VARCHAR(32), burntool_state SMALLINT, + video_cells TINYINT DEFAULT 0, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), @@ -1643,5 +1645,8 @@ workdir VARCHAR(255) NOT NULL, -- working directory comment VARCHAR(255), -- for human memory - PRIMARY KEY(client_id) + name varchar(64) default NULL, -- unique client_id verbose identifier + output_format SMALLINT NOT NULL default 1, -- format output versioning + PRIMARY KEY(client_id), + UNIQUE KEY name (name) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/eam_branches/ipp-20101205/ippTools/share/regtool_checkburntoolimfile.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/regtool_checkburntoolimfile.sql (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippTools/share/regtool_checkburntoolimfile.sql (revision 30102) @@ -0,0 +1,31 @@ +SELECT DISTINCT + exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + LEFT JOIN pzDownloadExp USING(exp_name) + LEFT JOIN newExp ON exp_name = tmp_exp_name + LEFT JOIN rawExp USING(exp_id, exp_name) + LEFT JOIN rawImfile USING(exp_name) + WHERE + summitExp.dateobs >= '@DATE@T00:00:00' + AND summitExp.dateobs <= '@DATE@T23:59:59' + AND (rawImfile.class_id = '@CLASS_ID@' OR rawImfile.class_id IS NULL) + AND (exp_name <= '@EXP_NAME@') +ORDER BY summitExp.dateobs Index: /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingburntoolimfile.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingburntoolimfile.sql (revision 30102) +++ /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingburntoolimfile.sql (revision 30102) @@ -0,0 +1,31 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + newImfile.tmp_class_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + LEFT JOIN pzDownloadExp USING(exp_name) + LEFT JOIN newExp ON exp_name = tmp_exp_name + LEFT JOIN newImfile USING(exp_id) + LEFT JOIN rawExp USING(exp_id, exp_name) + LEFT JOIN rawImfile ON (rawImfile.exp_name = newExp.tmp_exp_name AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitExp.dateobs >= '@DATE@T00:00:00' + AND summitExp.dateobs <= '@DATE@T23:59:59' +ORDER BY newImfile.tmp_class_id,summitExp.dateobs Index: /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingexp.sql =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingexp.sql (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingexp.sql (revision 30102) @@ -1,48 +1,54 @@ SELECT DISTINCT - exp_id, - tmp_exp_name, - tmp_camera, - tmp_telescope, - state, - workdir, - workdir_state, - reduction, - dvodb, - tess_id, - end_stage, - label, - camera, - filter, - obs_mode, - obs_group, - epoch, - dateobs -FROM - (SELECT - newExp.*, - newImfile.tmp_class_id, - rawImfile.tmp_class_id as raw_tmp_class_id, - rawImfile.camera, - rawImfile.filter, - rawImfile.dateobs, - rawImfile.obs_mode, - rawImfile.obs_group - FROM newExp - JOIN newImfile - USING(exp_id) - LEFT JOIN rawExp - USING(exp_id) - LEFT JOIN rawImfile - ON newImfile.exp_id = rawImfile.exp_id - AND newImfile.tmp_class_id = rawImfile.tmp_class_id - WHERE - newExp.state = 'run' - AND rawExp.exp_id IS NULL --- where hook %s - GROUP BY - newExp.exp_id - HAVING - COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id) - AND SUM(rawImfile.fault) = 0 + exp_id, + tmp_exp_name, + tmp_camera, + tmp_telescope, + state, + workdir, + workdir_state, + reduction, + dvodb, + tess_id, + end_stage, + label, + camera, + filter, + obs_mode, + obs_group, + epoch, + dateobs +FROM + ( + select DISTINCT * from + ( + select newExp.*,count(newImfile.tmp_class_id) AS new_count + from newExp + LEFT JOIN newImfile USING (exp_id) + LEFT JOIN rawExp USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + GROUP BY newExp.exp_id + ) AS NEWEXPOSURES + JOIN + ( + select DISTINCT newExp.exp_id, + rawImfile.camera, + rawImfile.filter, + rawImfile.dateobs, + rawImfile.obs_mode, + rawImfile.obs_group, + count(rawImfile.tmp_class_id) AS raw_count + from newExp + LEFT JOIN rawExp USING (exp_id) + LEFT JOIN rawImfile USING (exp_id) + where + newExp.state = 'run' AND rawExp.exp_id IS NULL + AND rawImfile.data_state = 'full' + -- where hook %s + GROUP BY newExp.exp_id + HAVING SUM(rawImfile.fault) = 0 + ) AS RAWEXPOSURES + USING (exp_id) + WHERE raw_count = new_count + ) AS Foo -- limit hook %s - ) as Foo Index: /branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/chiptool.c (revision 30102) @@ -271,7 +271,8 @@ // default + PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - + // find the exp_id of all the exposures that we want to queue up. psString query = pxDataGet("chiptool_find_rawexp.sql"); @@ -288,4 +289,8 @@ psFree(where); + if (unique) { + psStringAppend(&query, "AND chip_id IS NULL"); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/eam_branches/ipp-20101205/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/chiptoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/chiptoolConfig.c (revision 30102) @@ -61,4 +61,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous chipRun", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: /branches/eam_branches/ipp-20101205/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/difftool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/difftool.c (revision 30102) @@ -1978,4 +1978,5 @@ PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_data_group", "stackRun.data_group","=="); PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); Index: /branches/eam_branches/ipp-20101205/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/difftoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/difftoolConfig.c (revision 30102) @@ -336,4 +336,5 @@ psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL); psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL); + psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_data_group", 0, "search by stack data_group for input", NULL); psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); Index: /branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c (revision 30102) @@ -1155,6 +1155,6 @@ psMetadata *where = psMetadataAlloc(); - // new state - PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); + // new state (required + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); // old state (required) PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); Index: /branches/eam_branches/ipp-20101205/ippTools/src/magicdstoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/magicdstoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/magicdstoolConfig.c (revision 30102) @@ -163,5 +163,5 @@ // -clearstatefaults psMetadata *clearstatefaultsArgs = psMetadataAlloc(); - psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); Index: /branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c (revision 30102) @@ -950,9 +950,12 @@ PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + PXOPT_COPY_S32(config->args, where, "-fault", "pstampDependent.fault", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); - // if (fault_count) { - PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); - // } + PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">="); // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself. Index: /branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c (revision 30102) @@ -178,4 +178,6 @@ psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id of jobs to update", 0); psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id of jobs to update", 0); + psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL, "-stage_id", 0, "stage_id of jobs to update", 0); + psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-component", 0, "component of jobs to update", NULL); psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-fault", 0, "current value for dependent fault", 0); psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); Index: /branches/eam_branches/ipp-20101205/ippTools/src/pubtool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/pubtool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/pubtool.c (revision 30102) @@ -96,6 +96,8 @@ PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked", false); - - if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) { + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_S16(output_format, config->args, "-output_format", false, false); + + if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment, name, output_format)) { psError(PS_ERR_UNKNOWN, false, "Database error"); return false; @@ -313,4 +315,5 @@ PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "=="); PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-name", "publishClient.name", "LIKE"); pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "=="); Index: /branches/eam_branches/ipp-20101205/ippTools/src/pubtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/pubtoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/pubtoolConfig.c (revision 30102) @@ -83,4 +83,5 @@ psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-name", 0, "search on client name", NULL); // -add Index: /branches/eam_branches/ipp-20101205/ippTools/src/pztool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/pztool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/pztool.c (revision 30102) @@ -112,5 +112,5 @@ PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - + if (!pzDataStoreInsert(config->dbh, camera, Index: /branches/eam_branches/ipp-20101205/ippTools/src/regtool.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/regtool.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/regtool.c (revision 30102) @@ -34,4 +34,6 @@ static bool pendingimfileMode(pxConfig *config); static bool addprocessedimfileMode(pxConfig *config); +static bool checkburntoolimfileMode(pxConfig *config); +static bool pendingburntoolimfileMode(pxConfig *config); static bool processedimfileMode(pxConfig *config); static bool revertprocessedimfileMode(pxConfig *config); @@ -72,4 +74,6 @@ // imfile MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); + MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); @@ -176,4 +180,294 @@ } +static bool checkburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psString query = pxDataGet("regtool_checkburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,exp_name,"@EXP_NAME@"); + psStringSubstitute(&query,class_id,"@CLASS_ID@"); + psStringSubstitute(&query,date,"@DATE@"); + + fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + ok_to_burn = false; + } + if (already_burned == false) { + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + psFree(previous_uri); + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + return(true); + +} + +static bool pendingburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false); + psString query = pxDataGet("regtool_pendingburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,date,"@DATE@"); + + // fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + psString this_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + psString previous_class_id = NULL; + psString this_class_id = NULL; + psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + // Add the information about this row and the previous, if it exists. + // Write the class_id stuff for debugging. + this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&this_class_id,"XY","ota"); + psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id); + if (previous_class_id) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); + } + // class_id = NULL sorts to the top of the list, so skip those (incomplete downloads) + if (!this_class_id) { + continue; + } + // Determine if we've crossed a class_id boundary, as this resets the bits. + if (previous_class_id) { + if (strcmp(this_class_id,previous_class_id) != 0) { + ok_to_burn = true; + already_burned = true; + previous_uri = NULL; + } + } + + // Write the URIs as well. + this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging. + psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + } + + // Check the two status variables + // Convert bits of the SQL query into booleans describing the data state + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); + ok_to_burn = false; + } + if (already_burned == false) { + // printf("already_burned looks false %s %s\n",this_uri,this_class_id); + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + + // Check the uri for this exposure + if (!this_uri) { + ok_to_burn = false; + already_burned = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + + // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); + + // If the state of this imfile is not "pending_burntool" then we can't burn it. + if (!ignore_state) { + psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); + + if (!imfile_state) { // imfile state is NULL, so we probably aren't registered. + ok_to_burn = false; + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + } + + // Determine if we've already suggested an entry for this ota, and if not, copy this + // suggestion into our output result list. + if (!ok_to_burn || already_burned) { + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + // If we're here, then we think we could potentially burntool this file. + psArrayAdd(results,results->n,row); + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + psFree(results); + return false; + } + + psFree(output); + psFree(results); + + + return(true); + +} + + static bool addprocessedimfileMode(pxConfig *config) @@ -250,5 +544,6 @@ PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); - + PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); @@ -323,4 +618,5 @@ bytes, md5sum, + video_cells, 0 // burntool_state )) { @@ -809,4 +1105,7 @@ PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(chip_workdir,config->args, "-chip_workdir", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); @@ -1008,13 +1307,13 @@ // else continue on... - + chip_workdir = chip_workdir ? chip_workdir : workdir; // insert an entry into the chipPendingExp table // this can only be run as the newExp's state has been set to stop if (!pxchipQueueByExpTag(config, exp_id, - workdir, + chip_workdir, label, - NULL, // data_group - NULL, // dist_group + data_group, + dist_group, reduction, NULL, // expgroup @@ -1092,5 +1391,7 @@ PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">="); PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); - PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); + PXOPT_COPY_STR(config->args, where, "-object", "object", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); Index: /branches/eam_branches/ipp-20101205/ippTools/src/regtool.h =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/regtool.h (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/regtool.h (revision 30102) @@ -26,4 +26,6 @@ REGTOOL_MODE_NONE = 0x0, REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_CHECKBURNTOOLIMFILE, + REGTOOL_MODE_PENDINGBURNTOOLIMFILE, REGTOOL_MODE_ADDPROCESSEDIMFILE, REGTOOL_MODE_PROCESSEDIMFILE, Index: /branches/eam_branches/ipp-20101205/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20101205/ippTools/src/regtoolConfig.c (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippTools/src/regtoolConfig.c (revision 30102) @@ -53,4 +53,22 @@ ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); + // -checkburntoolimfile + psMetadata *checkburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, checkburntoolimfileArgs, "-exp_name", "define the exp_name (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Str, checkburntoolimfileArgs, "-inst", "define the camera name", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-telescope", "define the telescope name", NULL); + ADD_OPT(Bool, checkburntoolimfileArgs, "-simple", "use the simple output format", false); + + // -pendingburntoolimfile + psMetadata *pendingburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, pendingburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple", "use the simple output format", false); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state", "ignore the data_state when deciding what to work on", false); + ADD_OPT(U64, pendingburntoolimfileArgs, "-limit", "limit result set to N items", 0); + // -addprocessedimfile psMetadata *addprocessedimfileArgs = psMetadataAlloc(); @@ -128,4 +146,5 @@ ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); + ADD_OPT(Bool, addprocessedimfileArgs, "-video_cells", "define if chip has video cells", false); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -249,4 +268,7 @@ ADD_OPT(F32, addprocessedexpArgs, "-moon_phase", "define moon phase (0.0 = new)", NAN); ADD_OPT(Str, addprocessedexpArgs, "-label", "define label for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-data_group", "define data_group for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-dist_group", "define dist_group for chip stage (non-detrend data only)", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-chip_workdir", "define workdir for chip stage (non-detrend data only)", NULL); ADD_OPT(Str, addprocessedexpArgs, "-hostname", "define host name", NULL); ADD_OPT(S16, addprocessedexpArgs, "-fault", "set fault code", 0); @@ -291,4 +313,7 @@ psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by exposure obs_mode", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-comment", 0, "search by exposure comment", NULL); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); @@ -395,4 +420,6 @@ PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-checkburntoolimfile", "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs); + PXOPT_ADD_MODE("-pendingburntoolimfile", "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080925.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080925.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080925.config (revision 30102) @@ -235,4 +235,5 @@ CELL.YPARITY STR ATM2_2 + CHIP.VIDEOCELL STR CELLMODE # XY24 is bad, so get it from the database; all others from header CHIP.TEMP.DEPEND STR CHIP.NAME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080929.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080929.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20080929.config (revision 30102) @@ -230,4 +230,5 @@ CELL.YPARITY STR ATM2_2 + CHIP.VIDEOCELL STR CELLMODE # XY24 is bad, so get it from the database; all others from header CHIP.TEMP.DEPEND STR CHIP.NAME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20081011.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20081011.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20081011.config (revision 30102) @@ -221,4 +221,5 @@ CELL.YPARITY STR ATM2_2 + CHIP.VIDEOCELL STR CELLMODE # XY24 is bad, so get it from the database; all others from header CHIP.TEMP.DEPEND STR CHIP.NAME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090120.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090120.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090120.config (revision 30102) @@ -209,4 +209,5 @@ FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE FPA.PON.TIME STR PONTIME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090220.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090220.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20090220.config (revision 30102) @@ -209,4 +209,5 @@ FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE FPA.PON.TIME STR PONTIME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20100122.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20100122.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_20100122.config (revision 30102) @@ -209,4 +209,5 @@ FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE FPA.PON.TIME STR PONTIME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_orig.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_orig.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/format_orig.config (revision 30102) @@ -217,4 +217,5 @@ FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE FPA.PON.TIME STR PONTIME Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/ppImage.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/ppImage.config (revision 30102) @@ -13,5 +13,5 @@ NORM.CLASS STR CHIP # How to find the per-class normalizations -NONLIN BOOL TRUE # apply non-linearity correction +NONLIN BOOL FALSE # apply non-linearity correction OLDDARK BOOL FALSE @@ -280,5 +280,5 @@ CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? OVERSCAN BOOL TRUE # Overscan subtraction - NONLIN BOOL TRUE # apply non-linearity correction + NONLIN BOOL FALSE # apply non-linearity correction BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction @@ -310,5 +310,5 @@ CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? OVERSCAN BOOL TRUE # Overscan subtraction - NONLIN BOOL TRUE # apply non-linearity correction + NONLIN BOOL FALSE # apply non-linearity correction BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction Index: /branches/eam_branches/ipp-20101205/ippconfig/gpc1/psastro.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/gpc1/psastro.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/gpc1/psastro.config (revision 30102) @@ -291,2 +291,6 @@ REFSTAR_MASK BOOL FALSE END + +SAS_REFERENCE METADATA + PSASTRO.CATDIR STR SAS.REF.V1 +END Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/nightly_science.config (revision 30102) @@ -1,32 +1,36 @@ -RETENTION_TIME U16 9000 - CLEAN_MODES MULTI CLEAN_MODES METADATA - MODE STR CHIP - COMMAND STR chiptool + MODE STR CHIP + COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 30 END CLEAN_MODES METADATA - MODE STR WARP - COMMAND STR warptool + MODE STR WARP + COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 7 END CLEAN_MODES METADATA - MODE STR DIFF - COMMAND STR difftool + MODE STR DIFF + COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME S16 30 -# RETENTION_TIME S16 -1 -END -CLEAN_MODES METADATA - MODE STR DIST - COMMAND STR disttool +END +CLEAN_MODES METADATA + MODE STR DIST + COMMAND STR disttool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label @LABEL@ -time_stamp_end @CURRENT_DATE@ RETENTION_TIME S16 10 - ALTERNATE_CMD STR A -END -CLEAN_MODES METADATA - MODE STR MAGICDS - COMMAND STR magicdstool +END +CLEAN_MODES METADATA + MODE STR MAGICDS + COMMAND STR magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ RETENTION_TIME S16 1 - ALTERNATE_CMD STR B +END + +END_OF_NIGHT MULTI +END_OF_NIGHT METADATA + NAME STR MORNING_DARKS + OBSMODE STR ENGINEERING + EXPTYPE STR DARK + OBJECT STR ENGINEERING + COMMENT STR darks_morn% END @@ -43,8 +47,4 @@ MACRO_DEFINITIONS MULTI -#MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_chips -# MACRO STR echo survey.del.destreak @LABEL@ -#END MACRO_DEFINITIONS METADATA PROC_MODE STR queue_chips @@ -55,360 +55,199 @@ MACRO STR survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak END -#MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_confirm_stacks -# MACRO STR echo survey.add.destreak @LABEL@ /data/ipp053.0/gpc1_destreak -#END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR clean_old -# MACRO STR + +TARGETS MULTI +TARGETS METADATA + NAME STR M31 + DISTRIBUTION STR M31 + TESS STR M31 + OBSMODE STR M31 + OBJECT STR M31% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD01 + DISTRIBUTION STR MD01 + TESS STR MD01.V2 + OBSMODE STR MD + OBJECT STR MD01% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD02 + DISTRIBUTION STR MD02 + TESS STR MD02.V2 + OBSMODE STR MD + OBJECT STR MD02% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD03 + DISTRIBUTION STR MD03 + TESS STR MD03 + OBSMODE STR MD + OBJECT STR MD03% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD04 + DISTRIBUTION STR MD04 + TESS STR MD04.V2 + OBSMODE STR MD + OBJECT STR MD04% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD05 + DISTRIBUTION STR MD05 + TESS STR MD05 + OBSMODE STR MD + OBJECT STR MD05% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD06 + DISTRIBUTION STR MD06 + TESS STR MD06 + OBSMODE STR MD + OBJECT STR MD06% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD07 + DISTRIBUTION STR MD07 + TESS STR MD07 + OBSMODE STR MD + OBJECT STR MD07% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD08 + DISTRIBUTION STR MD08 + TESS STR MD08.V2 + OBSMODE STR MD + OBJECT STR MD08% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD09 + DISTRIBUTION STR MD09 + TESS STR MD09.V2 + OBSMODE STR MD + OBJECT STR MD09% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD10 + DISTRIBUTION STR MD10 + TESS STR MD10.V2 + OBSMODE STR MD + OBJECT STR MD10% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR MD11 + DISTRIBUTION STR MD11 + TESS STR MD11.V2 + OBSMODE STR MD + OBJECT STR MD11% + STACKABLE BOOL TRUE + MIN_STACK U16 8 + DIFFABLE BOOL FALSE + DIST S16 30 +END +TARGETS METADATA + NAME STR STS + DISTRIBUTION STR STS + TESS STR STS + OBSMODE STR STS% + OBJECT STR STS% + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE +END +TARGETS METADATA + NAME STR SweetSpot + DISTRIBUTION STR SweetSpot + TESS STR RINGS.V0 + OBSMODE STR SS + STACKABLE BOOL FALSE + DIFFABLE BOOL TRUE + WARP S16 60 +END +TARGETS METADATA + NAME STR OSS + DISTRIBUTION STR SweetSpot + TESS STR RINGS.V0 + OBSMODE STR OSS + STACKABLE BOOL FALSE + DIFFABLE BOOL TRUE +END +TARGETS METADATA + NAME STR CNP + DISTRIBUTION STR CNP + TESS STR RINGS.V0 + OBSMODE STR CNP + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE +END +TARGETS METADATA + NAME STR ThreePi + DISTRIBUTION STR ThreePi + TESS STR RINGS.V0 + OBSMODE STR 3PI + STACKABLE BOOL FALSE + DIFFABLE BOOL TRUE + CHIP S16 14 + DIFF S16 14 +END +# This is a hack, and I freely admit it. +# Removed the rest because we do not regularly run microtests. +# TARGETS METADATA +# NAME STR microtestMD01 +# TESS STR MD01 +# OBJECT STR MD01% +# COMMENT STR Microtest Obs% +# STACKABLE BOOL TRUE # END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_registration -# MACRO STR +# TARGETS METADATA +# NAME STR microtestMD01.noPattern +# TESS STR MD01 +# OBJECT STR MD01% +# COMMENT STR Microtest Obs% +# REDUCTION STR MICROTEST_NOPATTERN +# STACKABLE BOOL TRUE # END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR define_burntool -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_burntool -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_stacks -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_stacks -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_sweetspot -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_sweetspot -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_dqstats -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_dqstats -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR check_detrends -# MACRO STR -# END -# MACRO_DEFINITIONS METADATA -# PROC_MODE STR queue_detrends -# MACRO STR -# END - -TARGETS MULTI -TARGETS METADATA - NAME STR M31 - TESS STR M31 - OBSMODE STR M31 - OBJECT STR M31% -# COMMENT STR M31% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD01 - TESS STR MD01.V2 - OBSMODE STR MD - OBJECT STR MD01% -# COMMENT STR %MD01% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD02 - TESS STR MD02 - OBSMODE STR MD - OBJECT STR MD02% -# COMMENT STR %MD02% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD03 - TESS STR MD03 - OBSMODE STR MD - OBJECT STR MD03% -# COMMENT STR %MD03% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD04 - TESS STR MD04 - OBSMODE STR MD - OBJECT STR MD04% -# COMMENT STR %MD04% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD05 - TESS STR MD05 - OBSMODE STR MD - OBJECT STR MD05% -# COMMENT STR %MD05% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD06 - TESS STR MD06 - OBSMODE STR MD - OBJECT STR MD06% -# COMMENT STR %MD06% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD07 - TESS STR MD07 - OBSMODE STR MD - OBJECT STR MD07% -# COMMENT STR %MD07% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD08 - TESS STR MD08.V2 - OBSMODE STR MD - OBJECT STR MD08% -# COMMENT STR %MD08% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD09 - TESS STR MD09.V2 - OBSMODE STR MD - OBJECT STR MD09% -# COMMENT STR %MD09% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD10 - TESS STR MD10.V2 - OBSMODE STR MD - OBJECT STR MD10% -# COMMENT STR %MD10% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR MD11 - TESS STR MD11.V2 - OBSMODE STR MD - OBJECT STR MD11% -# COMMENT STR %MD11% - STACKABLE BOOL TRUE - DIST S16 30 -END -TARGETS METADATA - NAME STR STS - TESS STR STS - OBSMODE STR STS% - OBJECT STR STS% -# COMMENT STR Stellar Transit% - STACKABLE BOOL FALSE -END -TARGETS METADATA - NAME STR SweetSpot - TESS STR RINGS.V0 - OBSMODE STR SS -# OBJECT STR SS - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL TRUE -# NOCLEAN BOOL TRUE - WARP S16 60 -END -TARGETS METADATA - NAME STR OSS - DISTRIBUTION STR SweetSpot - TESS STR RINGS.V0 - OBSMODE STR OSS - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL TRUE -END -TARGETS METADATA - NAME STR CNP - DISTRIBUTION STR CNP - TESS STR RINGS.V0 - OBSMODE STR CNP - STACKABLE BOOL FALSE - EXTRA_PROCESSING BOOL FALSE -END -TARGETS METADATA - NAME STR ThreePi - TESS STR RINGS.V0 - OBSMODE STR 3PI -# OBJECT STR 3PI - STACKABLE BOOL FALSE - CHIP S16 14 - DIFF S16 14 -END -# This is a hack, and I freely admit it. -TARGETS METADATA - NAME STR microtestMD01 - TESS STR MD01 - OBJECT STR MD01% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD01.noPattern - TESS STR MD01 - OBJECT STR MD01% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD02 - TESS STR MD02 - OBJECT STR MD02% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD02.noPattern - TESS STR MD02 - OBJECT STR MD02% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD03 - TESS STR MD03 - OBJECT STR MD03% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD03.noPattern - TESS STR MD03 - OBJECT STR MD03% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD04 - TESS STR MD04 - OBJECT STR MD04% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD04.noPattern - TESS STR MD04 - OBJECT STR MD04% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD05 - TESS STR MD05 - OBJECT STR MD05% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD05.noPattern - TESS STR MD05 - OBJECT STR MD05% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD06 - TESS STR MD06 - OBJECT STR MD06% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD06.noPattern - TESS STR MD06 - OBJECT STR MD06% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD07 - TESS STR MD07 - OBJECT STR MD07% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD07.noPattern - TESS STR MD07 - OBJECT STR MD07% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD08 - TESS STR MD08 - OBJECT STR MD08% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD08.noPattern - TESS STR MD08 - OBJECT STR MD08% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD09 - TESS STR MD09 - OBJECT STR MD09% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD09.noPattern - TESS STR MD09 - OBJECT STR MD09% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD10 - TESS STR MD10 - OBJECT STR MD10% - COMMENT STR Microtest Obs% - STACKABLE BOOL TRUE -END -TARGETS METADATA - NAME STR microtestMD10.noPattern - TESS STR MD10 - OBJECT STR MD10% - COMMENT STR Microtest Obs% - REDUCTION STR MICROTEST_NOPATTERN - STACKABLE BOOL TRUE -END # Detrend verification data @@ -420,6 +259,5 @@ REF_ID S64 299 REF_ITER S32 0 -# FILTER STR - MAX S32 10 + MAX_EXP S32 10 END @@ -431,5 +269,5 @@ REF_ITER S32 0 FILTER STR g.00000 - MAX S32 10 + MAX_EXP S32 10 END @@ -441,5 +279,5 @@ REF_ITER S32 0 FILTER STR r.00000 - MAX S32 10 + MAX_EXP S32 10 END @@ -451,5 +289,5 @@ REF_ITER S32 0 FILTER STR i.00000 - MAX S32 10 + MAX_EXP S32 10 END @@ -461,5 +299,5 @@ REF_ITER S32 0 FILTER STR z.00000 - MAX S32 10 + MAX_EXP S32 10 END @@ -471,5 +309,5 @@ REF_ITER S32 0 FILTER STR y.00000 - MAX S32 10 + MAX_EXP S32 10 END @@ -481,5 +319,5 @@ REF_ITER S32 0 FILTER STR w.00000 - MAX S32 10 -END - + MAX_EXP S32 10 +END + Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppImage.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppImage.config (revision 30102) @@ -947,4 +947,6 @@ BIN1.JPEG BOOL TRUE # Save 1st binned jpeg? BIN2.JPEG BOOL TRUE # Save 2nd binned jpeg? + MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels + MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels END @@ -977,4 +979,6 @@ BIN2.XBIN S32 1 # Image is already binned BIN2.YBIN S32 1 # Image is already binned + MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels + MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels END @@ -1007,4 +1011,6 @@ BIN2.XBIN S32 1 # Image is already binned BIN2.YBIN S32 1 # Image is already binned + MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels + MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels END Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStats.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStats.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStats.config (revision 30102) @@ -236,5 +236,8 @@ CONCEPT STR FPA.BURNTOOL.APPLIED + CONCEPT STR FPA.PON.TIME # time since last power on + + CONCEPT STR CHIP.VIDEOCELL STAT STR ROBUST_MEDIAN # Background estimator Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStatsFromMetadata.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/ppStatsFromMetadata.config (revision 30102) @@ -45,4 +45,5 @@ ENTRY VAL FPA.PON.TIME F32 CONSTANT -pon_time # time since last power on # ENTRY VAL FPA.BURNTOOL.APPLIED S32 CONSTANT -burntool_state # + ENTRY VAL CHIP.VIDEOCELL BOOL CONSTANT -video_cells # This chip contains a video cell ENTRY VAL CHIP.TEMP F32 SAMPLE_MEAN -ccd_temp # CCD temperature ENTRY VAL CELL.EXPOSURE F32 SAMPLE_MEAN -exp_time # Exposure time Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/psastro.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/psastro.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/psastro.config (revision 30102) @@ -238,2 +238,6 @@ MOPS.TEST METADATA END + +SAS_REFERENCE METADATA +END + Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/psphot.config (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/psphot.config (revision 30102) @@ -49,4 +49,5 @@ PEAKS_NMAX S32 5000 # on first pass, only keep NMAX peaks (0 == all) PEAKS_MIN_GAUSS F32 0.5 # On quick convolution, mask pixels for which the +PEAKS_POS2_NSIGMA_LIMIT F32 25.0 # peak signficance threshold for POS2 sources. (ppSub) # input pixels contribute less than this fraction of the flux # parameters which adjust the footprint analysis Index: /branches/eam_branches/ipp-20101205/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20101205/ippconfig/recipes/reductionClasses.mdc (revision 30101) +++ /branches/eam_branches/ipp-20101205/ippconfig/recipes/reductionClasses.mdc (revision 30102) @@ -543,2 +543,19 @@ END +SAS_REFERENCE METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR SAS_REFERENCE + STACKPHOT STR STACKPHOT +END + Index: /branches/eam_branches/ipp-20101205/tools/czarpoll.pl =================================================================== --- /branches/eam_branches/ipp-20101205/tools/czarpoll.pl (revision 30101) +++ /branches/eam_branches/ipp-20101205/tools/czarpoll.pl (revision 30102) @@ -108,50 +108,51 @@ my $newState = undef; my $nsStatus = undef; - my $lastDay = strftime('%Y-%m-%d', localtime); my $today = undef; - my $doneMetricsToday = 1; - + my $yesterday = undef; + my $newDayTime = "18:00"; + my $lastDayDailyTasks = "2010-01-01"; + + # main polling loop while (1) { - # check whether day has changed. if so, cleanup tables from previous day and optimize + # sort out times $today = strftime('%Y-%m-%d', localtime); - if ($czarDb->isBefore($lastDay, $today)) { - + $yesterday = $czarDb->subtractInterval($today, "1 DAY"); + $end = $czarDb->getNowTimestamp(); + + # if before 18:00 today, then start plots from 18:00 yesterday + if ($czarDb->isBefore($end, "$today $newDayTime")) {$begin = "$yesterday $newDayTime";} + # if after 18:00 today, then perform some daily tasks and start plots from 18:00 today + else { + + $begin = "$today $newDayTime"; + + # check whether yesterday was cleaned. if not, cleanup tables and optimize + if ($lastDayDailyTasks ne $yesterday) { + + print "* performing daily tasks (clean-up, metrics)"; + + # create metrics for last 24 hours + print "* Creating metrics for last 24 hours\n"; + # TODO hardcoded path needs to be in config + my $dayMetrics = new czartool::DayMetrics($gpc1Db, + $czarDb, + "/data/ipp004.0/ipp/ippMetrics/", + 1, 0, $today); + $dayMetrics->writeHTML(); + + # now update metrics index page + my $metricsIndex = new czartool::MetricsIndex($gpc1Db, + $czarDb, + "/data/ipp004.0/ipp/ippMetrics/", + 1, 0); + $metricsIndex->writeHTML(); + + # now cleanup tables from yesterday and optimize print "* New day - performing cleanup\n"; - $czarDb->cleanupDateRange($lastDay, $lastDay, "30 MINUTE"); + $czarDb->cleanupDateRange($yesterday, $yesterday, "30 MINUTE"); $czarDb->optimize(); - $lastDay = $today; - $doneMetricsToday = 0; - } - - # sort out times - $begin = strftime('%Y-%m-%d 06:30', localtime); - $end = $czarDb->getNowTimestamp(); - - # if time now is after 06:35am, then create metrics for past 24 hours - if (!$doneMetricsToday && $czarDb->isBefore($begin, $end)) { - - # create metrics for last 24 hours - print "* Creating metrics for last 24 hours\n"; - my $yesterday = $czarDb->subtractInterval($today, "1 DAY"); - # TODO hardcoded path needs to be in config - my $dayMetrics = new czartool::DayMetrics($gpc1Db, - $czarDb, - "/data/ipp004.0/ipp/ippMetrics/", - 1, 0, $yesterday); - $dayMetrics->writeHTML(); - $doneMetricsToday = 1; - - # now update metrics index page - my $metricsIndex = new czartool::MetricsIndex($gpc1Db, - $czarDb, - "/data/ipp004.0/ipp/ippMetrics/", - 1, 0); - } - - # if time now is before 06:35am, include data from previous day - if ($czarDb->isBefore($end, $begin)) { - - $begin = $czarDb->subtractInterval($begin, "1 DAY"); + $lastDayDailyTasks = $yesterday; + } } @@ -247,5 +248,5 @@ $plotter->createLogAndLinearTimeSeries($allServerLabels, $stage, $begin, $end); # TODO must be a neater way... - $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0); + $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0); } } @@ -271,5 +272,5 @@ my $server = undef; my $state = undef; - + foreach $stage (@stages) { Index: /branches/eam_branches/ipp-20101205/tools/czartool/DayMetrics.pm =================================================================== --- /branches/eam_branches/ipp-20101205/tools/czartool/DayMetrics.pm (revision 30101) +++ /branches/eam_branches/ipp-20101205/tools/czartool/DayMetrics.pm (revision 30102) @@ -37,6 +37,8 @@ $self->{day} = $_[6]; + my $yesterday = $self->{czarDb}->subtractInterval($self->{day}, "1 DAY"); + # sort out times - $self->{begin} = "$self->{day} 06:00"; + $self->{begin} = "$yesterday 18:00"; $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY"); $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR"); Index: /branches/eam_branches/ipp-20101205/tools/czartool/Metrics.pm =================================================================== --- /branches/eam_branches/ipp-20101205/tools/czartool/Metrics.pm (revision 30101) +++ /branches/eam_branches/ipp-20101205/tools/czartool/Metrics.pm (revision 30102) @@ -35,9 +35,7 @@ # instantiate a plotter object - $self->{plotter} = new czartool::Plotter( + $self->{plotter} = czartool::Plotter->new_file( $self->{gpc1Db}, $self->{czarDb}, - "%Y%m%d-%H%M%S", - "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", ".", $self->{save_temps}); Index: /branches/eam_branches/ipp-20101205/tools/makedistdest =================================================================== --- /branches/eam_branches/ipp-20101205/tools/makedistdest (revision 30101) +++ /branches/eam_branches/ipp-20101205/tools/makedistdest (revision 30102) @@ -13,13 +13,16 @@ my $prod_name; +my $dbhost; GetOptions( 'prod_name=s' => \$prod_name, + 'dbhost=s' => \$dbhost, ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --prod_name", +pod2usage( -msg => "Required options: --prod_name --dbhost", -exitval => 3) - unless defined $prod_name; + unless defined $prod_name + and defined $dbhost; my $cmd = "dsprodtool --add $prod_name --type ipp-dist --description $prod_name"; @@ -31,5 +34,5 @@ die "dsprodtool failed: $rc" if $rc; -$cmd = "disttool -dbname gpc1 -definedestination -ds_dbname ippRequestServer -ds_dbhost ippdb02 -name $prod_name"; +$cmd = "disttool -dbname gpc1 -definedestination -ds_dbname ippRequestServer -ds_dbhost $dbhost -name $prod_name"; $rc = system $cmd;