Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37277) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl (revision 37278) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single chipRun use Carp; @@ -16,24 +18,12 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -# my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. -my $threads = 2; # How many threads are we going to use? -my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for - -# We need to ensure we only ever try to transfer a file once. -my %file_filter = (); +my $threads_req = 4; # Look for programs we need my $missing_tools; -my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); @@ -46,7 +36,8 @@ # Options -my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose); GetOptions( 'remote_id=s' => \$remote_id, + 'chip_id=s' => \$chip_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -57,105 +48,29 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($chip_id) and defined($camera) and defined($path_base) and defined($dbname); + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1a: Load up the catalog of detrends -# Detrend concept holders -my %detrends = (); -my %det_types = ('MASK' => '-mask', - 'FLAT' => '-flat', - 'DARK' => '-dark', - 'VIDEODARK' => '-dark', - 'LINEARITY' => '-linearity', - 'FRINGE' => '-fringe', - 'NOISEMAP' => '-noisemap'); - -foreach my $det_type (keys (%det_types)) { - my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; - $dt_cmd .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $dt_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("UNable to run detselect to determine detrend catalog", - $remote_id,$error_code); - } - my $detData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine detrend information.", - $remote_id,$PS_EXIT_PROG_ERROR); - my $detData2= parse_md_list($detData); - - # Data structure for this: - # $detrend{NAME} = \@list_of_detselect_shows - # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) - # $detrend{NAME}[0]{class_id} = uri - - # At this point, I have the list of detselect shows, assign it - $detrends{$det_type} = $detData2; - - for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { - my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; - my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; - - my $detselect_command2 = "detselect -select "; - $detselect_command2 .= " -det_id $det_id "; - $detselect_command2 .= " -iteration $det_iter "; - $detselect_command2 .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $detselect_command2, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - } - - my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - my $detImfile2= parse_md_list($detImfile); - foreach $detImfile (@$detImfile2) { - my $class_id = $detImfile->{class_id}; - my $duri = $detImfile->{uri}; - - ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; - } - } -} - -#print Dumper(%detrends); +# Detrend concept holders (name, ppImage option) +my %det_types = ( + 'MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap' + ); my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN"); -# -# Step 1b: Open output files +# STEP 0: Open output files my $uri_command = $path_base . ".cmd"; my $uri_transfer= $path_base . ".transfer"; @@ -174,13 +89,10 @@ my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); - - -# -# Step 2: Iterate over all componenets in this remote run. +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id, $PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $chip_id, $PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $chip_id, $PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id, $PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); + my $job_index = 0; my @pre_commands = (); @@ -188,13 +100,11 @@ my @post_commands = (); -foreach my $compEntry (@$compData2) { - - my $chip_id = $compEntry->{stage_id}; - my $exp_id ; -# Get exposure level information from the chipRun we're working from. +# STEP 1 : Get exposure level information for this chipRun +my $exp_id, $filter, $altfilt, $dateobs; +{ my $command = "$chiptool -listrun -chip_id $chip_id "; $command .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); unless ($success) { @@ -205,72 +115,71 @@ } -# Parse chipRun level data to determine detrend information - my %detrends_to_use = (); - { - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine chip component information.", - $chip_id,$PS_EXIT_PROG_ERROR); - my $chipData2= parse_md_list($chipData); - my $chipEntry = ${ $chipData2 }[0]; - - $exp_id = $chipEntry->{exp_id}; - my $filter = $chipEntry->{filter}; - my $altfilt= $filter; - $altfilt =~ s/.00000//; - my $dateobs= $chipEntry->{dateobs}; - my ($date, $time) = split /T/, $dateobs; - my ($year,$month,$day) = split /-/, $date; - my ($hour,$minute,$second) = split /:/, $time; #/; - my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); - - foreach my $det_type (%detrends) { - if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y - - foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options - if (defined($det_obj->{filter})) { - if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { - next; # Skip detrends with a filter that we can't use. - } - } - if (defined($det_obj->{time_begin})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { - next; # Skip detrends that come into effect after this exposure (d1 < d2) - } - } - if (defined($det_obj->{time_end})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { - next; # Skip detrends that stop working before this exposure (d1 > d2) - } - } - $detrends_to_use{$det_type} = $det_obj; - last; - } - } - } - -# -# Step 3: Iterate over the sub-components -# Iterate over the chipProcessedImfile level data. - my $reg_command = "$chiptool -pendingimfile "; - $reg_command .= " -chip_id $chip_id " if defined($chip_id); - $reg_command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $reg_command, verbose => $verbose); + # Parse chipRun level data to determine exposure information + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", $chip_id,$PS_EXIT_PROG_ERROR); + my $metadata = parse_md_list($MDlist); + my $chipEntry = $metadata[0]; + + $exp_id = $chipEntry->{exp_id}; + $filter = $chipEntry->{filter}; + $altfilt = $filter; + $altfilt =~ s/.00000//; + $dateobs = $chipEntry->{dateobs}; +} + +# STEP 2 : select the det_ids for each desired det_type (appropriate to this exposure) +my %det_ids = (); +my %det_iters = (); + +foreach my $det_type (keys (%det_types)) { + if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + my $command = "detselect -search -det_type $det_type -time $dateobs"; + $command .= " -dbname $dbname " if defined($dbname); + + if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { + $command .= " -filter $altfilt"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("No valid detrend available for this image: $det_type", $chip_id, $error_code); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type", $chip_id,$error_code); + my $metadata = parse_md_list($MDlist); + my $detEntry = $metadata[0]; + + # save det_id and iteration + $det_ids{$det_type} = $detEntry->{det_id}; + $det_iters{$det_type} = $detEntry->{iteration}; +} + +# Step 3: Iterate over the sub-components +{ + # select chipProcessedImfile entries + my $command = "$chiptool -pendingimfile -chip_id $chip_id"; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run chiptool -pendingimfile", $chip_id,$error_code); } - - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - next; -# &my_die("Unable to parse chiptool -pendingimfile information.", -# $chip_id,$error_code); - my $chipData2= parse_md_list($chipData); - - - foreach my $chipEntry (@$chipData2) { + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse chiptool -pendingimfile information.", $chip_id,$error_code); + my $metadata = parse_md_list($MDlist); + + # Iterate over the chipProcessedImfile level data. + foreach my $chipEntry (@$metadata) { # Get information we need to pass to ppImage my $uri = $chipEntry->{uri}; @@ -283,8 +192,8 @@ # Process the image and burntool table - my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri); + my ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri); my $btt = $uri; $btt =~ s/fits$/burn.tbl/; - my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt); + my ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt); # Initialize the ppI command @@ -292,16 +201,38 @@ $ppImage_command .= " -burntool $remote_btt "; - # Add detrend information to the command line - foreach my $det (keys %detrends_to_use) { - if ((($video_cells)&&($det eq 'DARK'))|| - ((!$video_cells)&&($det eq 'VIDEODARK'))) { - next; + foreach my $det_type (keys (%det_types)) { + if (( $video_cells) && ($det_type eq 'DARK')) next; + if ((!$video_cells) && ($det_type eq 'VIDEODARK')) next; + + my $det_id = $det_ids{$det_type}; + my $det_iter = $det_iters{$det_type}; + + # Add detrend information to the command line + my $detselect_command = "detselect -select "; + $detselect_command .= " -det_id $det_id"; + $detselect_command .= " -iteration $det_iter"; + $detselect_command .= " -class_id $class_id"; + $detselect_command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command, verbose => 0); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); } - my $det_code = $det_types{$det}; - my $duri = $detrends_to_use{$det}->{$class_id}; + + my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + my $detImfileMD = parse_md_list($detMDlist); + my $duri = $detImfileMD[0]->{uri}; + + my $det_code = $det_types{$det_type}; my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); $ppImage_command .= " $det_code $remote_det_uri "; } - # Add output root @@ -318,5 +249,5 @@ $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; - $ppImage_command .= " -threads 4 "; + $ppImage_command .= " -threads $threads_req "; $ppImage_command .= " -image_id $chip_imfile_id "; $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; @@ -342,5 +273,4 @@ print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; - # Determine which output files need to be returned. @@ -354,29 +284,7 @@ # die; } - print CONFIG "\n"; } } - -# Actually put the commands into the output file. -# my %unique_cmds = (); -# foreach my $cmd (@pre_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@main_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@post_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } close(CONFIG); @@ -386,42 +294,4 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} -$time_req += 1; # Safety addition. - -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; -#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -close(COMMAND); - - ## We're done here. The execution and handling are done elsewhere. # Quick review: @@ -429,6 +299,8 @@ # auth unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $chip_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; $command .= " -dbname $dbname " if defined $dbname; @@ -436,8 +308,5 @@ } - - - - +exit (0); sub uri_convert { @@ -537,6 +406,9 @@ carp($msg); + I need stage id here + + if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; + my $command = "remotetool -updatecomponent -remote_id $id"; $command .= " -fault $exit_code " if defined $exit_code; $command .= " -set_state $exit_state " if defined $exit_state; Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl =================================================================== --- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37278) +++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl (revision 37278) @@ -0,0 +1,555 @@ +#! /usr/bin/env perl + +# this script takes a list of run components ready to be run +# (remoteComponent.state=prep_done) and generates the master run +# scripts for that set. Components which have +# (remoteComponent.state=prep_fail) are skipped + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +# my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +$remote_root = '/scratch3/watersc1/'; +my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. +my $threads = 2; # How many threads are we going to use? +my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for + +# We need to ensure we only ever try to transfer a file once. +my %file_filter = (); + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1a: Load up the catalog of detrends +# Detrend concept holders +my %detrends = (); +my %det_types = ('MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap'); + +foreach my $det_type (keys (%det_types)) { + my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; + $dt_cmd .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $dt_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("UNable to run detselect to determine detrend catalog", + $remote_id,$error_code); + } + my $detData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine detrend information.", + $remote_id,$PS_EXIT_PROG_ERROR); + my $detData2= parse_md_list($detData); + + # Data structure for this: + # $detrend{NAME} = \@list_of_detselect_shows + # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) + # $detrend{NAME}[0]{class_id} = uri + + # At this point, I have the list of detselect shows, assign it + $detrends{$det_type} = $detData2; + + for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { + my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; + my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; + + my $detselect_command2 = "detselect -select "; + $detselect_command2 .= " -det_id $det_id "; + $detselect_command2 .= " -iteration $det_iter "; + $detselect_command2 .= " -dbname $dbname " if defined($dbname); + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command2, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + } + + my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + my $detImfile2= parse_md_list($detImfile); + foreach $detImfile (@$detImfile2) { + my $class_id = $detImfile->{class_id}; + my $duri = $detImfile->{uri}; + + ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; + } + } +} + +#print Dumper(%detrends); + +my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN"); + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; +my $uri_generate= $path_base . ".generate"; +my $uri_return = $path_base . ".return"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); +my $disk_generate= $ipprc->file_resolve($uri_generate,1); +my $disk_return = $ipprc->file_resolve($uri_return,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); + + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); + +foreach my $compEntry (@$compData2) { + + my $chip_id = $compEntry->{stage_id}; + my $exp_id ; +# Get exposure level information from the chipRun we're working from. + my $command = "$chiptool -listrun -chip_id $chip_id "; + $command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $chip_id,$error_code); + } + +# Parse chipRun level data to determine detrend information + my %detrends_to_use = (); + { + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", + $chip_id,$PS_EXIT_PROG_ERROR); + my $chipData2= parse_md_list($chipData); + my $chipEntry = ${ $chipData2 }[0]; + + $exp_id = $chipEntry->{exp_id}; + my $filter = $chipEntry->{filter}; + my $altfilt= $filter; + $altfilt =~ s/.00000//; + my $dateobs= $chipEntry->{dateobs}; + my ($date, $time) = split /T/, $dateobs; + my ($year,$month,$day) = split /-/, $date; + my ($hour,$minute,$second) = split /:/, $time; #/; + my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); + + foreach my $det_type (%detrends) { + if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options + if (defined($det_obj->{filter})) { + if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { + next; # Skip detrends with a filter that we can't use. + } + } + if (defined($det_obj->{time_begin})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { + next; # Skip detrends that come into effect after this exposure (d1 < d2) + } + } + if (defined($det_obj->{time_end})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { + next; # Skip detrends that stop working before this exposure (d1 > d2) + } + } + $detrends_to_use{$det_type} = $det_obj; + last; + } + } + } + +# +# Step 3: Iterate over the sub-components +# Iterate over the chipProcessedImfile level data. + my $reg_command = "$chiptool -pendingimfile "; + $reg_command .= " -chip_id $chip_id " if defined($chip_id); + $reg_command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $reg_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool -pendingimfile", + $chip_id,$error_code); + } + + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + next; +# &my_die("Unable to parse chiptool -pendingimfile information.", +# $chip_id,$error_code); + my $chipData2= parse_md_list($chipData); + + + foreach my $chipEntry (@$chipData2) { + # Get information we need to pass to ppImage + my $uri = $chipEntry->{uri}; + my $class_id = $chipEntry->{class_id}; + my $video_cells = $chipEntry->{video_cells}; + my $reduction = $chipEntry->{reduction}; + my $exp_tag = $chipEntry->{exp_tag}; + my $workdir = $chipEntry->{workdir}; + my $chip_imfile_id = $chipEntry->{chip_imfile_id}; + + # Process the image and burntool table + my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri); + my $btt = $uri; + $btt =~ s/fits$/burn.tbl/; + my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt); + + # Initialize the ppI command + my $ppImage_command = "ppImage -file $remote_uri"; + $ppImage_command .= " -burntool $remote_btt "; + + # Add detrend information to the command line + foreach my $det (keys %detrends_to_use) { + if ((($video_cells)&&($det eq 'DARK'))|| + ((!$video_cells)&&($det eq 'VIDEODARK'))) { + next; + } + my $det_code = $det_types{$det}; + my $duri = $detrends_to_use{$det}->{$class_id}; + my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); + $ppImage_command .= " $det_code $remote_det_uri "; + } + + + # Add output root + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + $ppImage_command .= " $remote_outroot "; + print STDERR "$remote_outroot $ipp_outroot $class_id\n"; + # Complete reduction information. + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage + my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot + + $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; + $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; + $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; + $ppImage_command .= " -threads 4 "; + $ppImage_command .= " -image_id $chip_imfile_id "; + $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; + $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; + +# push @main_commands, $ppImage_command; + + # Calculate pre and post commands + my $remote_outroot_dir = dirname($remote_outroot); + my $pre_command = "mkdir -p $remote_outroot_dir"; + + my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; + $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; + $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; + $post_commandA .= " -dbname $dbname " if defined $dbname; + + my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; + my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; + +# push @post_commands, "$post_commandB $post_commandC"; + + $job_index++; + + print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; + + + # Determine which output files need to be returned. + foreach my $component (@return_component_list) { +# uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) ); + my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; +# print " $filename $ipp_disk $remote_disk\n"; +# die; + } + + print CONFIG "\n"; + } +} + +# Actually put the commands into the output file. +# my %unique_cmds = (); +# foreach my $cmd (@pre_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } +# foreach my $cmd (@main_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } +# foreach my $cmd (@post_commands) { +# unless(exists($unique_cmds{$cmd})) { +# print CONFIG $cmd . "\n"; +# $unique_cmds{$cmd} = 1; +# } +# } + +close(CONFIG); +close(TRANSFER); +close(CHECK); +close(RETURN); +close(GENERATE); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} +$time_req += 1; # Safety addition. + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +close(COMMAND); + + +## We're done here. The execution and handling are done elsewhere. +# Quick review: +# new -> pending -> run -> full +# auth +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + + + + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_convert_and_create { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_raw { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + $remote_disk = $remote_raw . $ipp_disk; + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs_for_return { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -remote_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} +