Index: trunk/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- trunk/ippScripts/scripts/sc_prepare_chip.pl (revision 37125) +++ trunk/ippScripts/scripts/sc_prepare_chip.pl (revision 37833) @@ -1,3 +1,5 @@ #! /usr/bin/env perl + +# generate the input & output files lists and commands for a single chipRun use Carp; @@ -16,27 +18,20 @@ use Pod::Usage qw( pod2usage ); - # Hard coded values -# my $remote_root = '/scratch3/watersc1/'; # Far side destination base location -my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location -$remote_root = '/scratch3/watersc1/'; -my $remote_raw = "${remote_root}/raw/"; # Directory to find raw data in. -my $threads = 2; # How many threads are we going to use? -my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. -my $proc_per_node = 24; # processors per node -my $min_nodes = 1; # smallest allocation to ask for -my $max_nodes = 1000; # largest allocation to ask for -my $min_time = 1; # shortest allocation to ask for -my $max_time = 8; # longest allocation to ask for - -# We need to ensure we only ever try to transfer a file once. -my %file_filter = (); +# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $remote_hostname = "LANL/Mustang"; # Name of the remote node. +my $remote_raw = "${remote_root}/tmp/"; # Directory to find raw data in. +my $threads_req = 4; + +my $fail_state = "prep_fail"; +my $hard_fail_state = "fail"; # Look for programs we need my $missing_tools; -my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); +my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1); if ($missing_tools) { @@ -46,7 +41,8 @@ # Options -my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose); GetOptions( 'remote_id=s' => \$remote_id, + 'chip_id=s' => \$chip_id, 'camera|c=s' => \$camera, 'dbname|d=s' => \$dbname, @@ -54,109 +50,33 @@ 'no_update' => \$no_update, 'verbose' => \$verbose, + 'dbverbose' => \$dbverbose, ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless +pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless defined($remote_id) and + defined($chip_id) and defined($camera) and defined($path_base) and defined($dbname); -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -# -# Step 1: Get a list of the components that make up this remoteRun - -# SHould this call listrun to ensure we're in state new? -my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; -$rt_cmd .= " -dbname $dbname " if defined($dbname); - -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $rt_cmd, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool to determine stage parameters.", - $remote_id,$error_code); -} -my $compData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine component information.", - $remote_id,$PS_EXIT_PROG_ERROR); -my $compData2= parse_md_list($compData); - -# -# Step 1a: Load up the catalog of detrends -# Detrend concept holders -my %detrends = (); -my %det_types = ('MASK' => '-mask', - 'FLAT' => '-flat', - 'DARK' => '-dark', - 'VIDEODARK' => '-dark', - 'LINEARITY' => '-linearity', - 'FRINGE' => '-fringe', - 'NOISEMAP' => '-noisemap'); - -foreach my $det_type (keys (%det_types)) { - my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; - $dt_cmd .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $dt_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("UNable to run detselect to determine detrend catalog", - $remote_id,$error_code); - } - my $detData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine detrend information.", - $remote_id,$PS_EXIT_PROG_ERROR); - my $detData2= parse_md_list($detData); - - # Data structure for this: - # $detrend{NAME} = \@list_of_detselect_shows - # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) - # $detrend{NAME}[0]{class_id} = uri - - # At this point, I have the list of detselect shows, assign it - $detrends{$det_type} = $detData2; - - for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { - my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; - my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; - - my $detselect_command2 = "detselect -select "; - $detselect_command2 .= " -det_id $det_id "; - $detselect_command2 .= " -iteration $det_iter "; - $detselect_command2 .= " -dbname $dbname " if defined($dbname); - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $detselect_command2, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - } - - my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", - $chip_id,$error_code); - my $detImfile2= parse_md_list($detImfile); - foreach $detImfile (@$detImfile2) { - my $class_id = $detImfile->{class_id}; - my $duri = $detImfile->{uri}; - - ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; - } - } -} - -#print Dumper(%detrends); +# Detrend concept holders (name, ppImage option) +my %det_types = ( + 'MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap' + ); my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN"); -# -# Step 1b: Open output files -my $uri_command = $path_base . ".cmd"; +# STEP 0: Open output files my $uri_transfer= $path_base . ".transfer"; my $uri_check = $path_base . ".check"; @@ -165,5 +85,4 @@ my $uri_return = $path_base . ".return"; -my $disk_command = $ipprc->file_resolve($uri_command,1); my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); my $disk_check = $ipprc->file_resolve($uri_check,1); @@ -174,209 +93,230 @@ my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. -open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); -open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); -open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); -open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR); -open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR); - - -# -# Step 2: Iterate over all componenets in this remote run. +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); +open(RETURN, ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); + my $job_index = 0; -my @pre_commands = (); -my @main_commands = (); -my @post_commands = (); - -foreach my $compEntry (@$compData2) { - - my $chip_id = $compEntry->{stage_id}; - my $exp_id ; -# Get exposure level information from the chipRun we're working from. + +# STEP 1 : Get exposure level information for this chipRun +my ($exp_id, $filter, $altfilt, $dateobs); +{ my $command = "$chiptool -listrun -chip_id $chip_id "; $command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool to determine stage parameters.", - $chip_id,$error_code); - } - -# Parse chipRun level data to determine detrend information - my %detrends_to_use = (); - { - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine chip component information.", - $chip_id,$PS_EXIT_PROG_ERROR); - my $chipData2= parse_md_list($chipData); - my $chipEntry = ${ $chipData2 }[0]; - - $exp_id = $chipEntry->{exp_id}; - my $filter = $chipEntry->{filter}; - my $altfilt= $filter; - $altfilt =~ s/.00000//; - my $dateobs= $chipEntry->{dateobs}; - my ($date, $time) = split /T/, $dateobs; - my ($year,$month,$day) = split /-/, $date; - my ($hour,$minute,$second) = split /:/, $time; #/; - my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); - - foreach my $det_type (%detrends) { - if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y - - foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options - if (defined($det_obj->{filter})) { - if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { - next; # Skip detrends with a filter that we can't use. - } - } - if (defined($det_obj->{time_begin})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { - next; # Skip detrends that come into effect after this exposure (d1 < d2) - } - } - if (defined($det_obj->{time_end})) { - if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { - next; # Skip detrends that stop working before this exposure (d1 > d2) - } - } - $detrends_to_use{$det_type} = $det_obj; - last; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state); + } + + # Parse chipRun level data to determine exposure information + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + my $chipEntry = $metadata->[0]; + + $exp_id = $chipEntry->{exp_id}; + $filter = $chipEntry->{filter}; + $altfilt = $filter; + $altfilt =~ s/.00000//; + $dateobs = $chipEntry->{dateobs}; +} + +# STEP 2 : select the det_ids for each desired det_type (appropriate to this exposure) +my %det_ids = (); +my %det_iters = (); + +foreach my $det_type (keys (%det_types)) { + if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + my $command = "detselect -search -det_type $det_type -time $dateobs"; + $command .= " -dbname $dbname " if defined($dbname); + + if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { + $command .= " -filter $altfilt"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + my $detEntry = $metadata->[0]; + + if ($verbose) { + print STDERR "det_id: $detEntry->{det_id}\n"; + print STDERR "det_iter: $detEntry->{iteration}\n"; + } + + # save det_id and iteration + $det_ids{$det_type} = $detEntry->{det_id}; + $det_iters{$det_type} = $detEntry->{iteration}; +} + +# Step 3: Iterate over the sub-components +{ + # select chipProcessedImfile entries + my $command = "$chiptool -pendingimfile -chip_id $chip_id"; + $command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $dbverbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $metadata = parse_md_list($MDlist); + + # Iterate over the chipProcessedImfile level data. + foreach my $chipEntry (@$metadata) { + # Get information we need to pass to ppImage + my $uri = $chipEntry->{uri}; + my $class_id = $chipEntry->{class_id}; + my $video_cells = $chipEntry->{video_cells}; + my $reduction = $chipEntry->{reduction}; + my $exp_tag = $chipEntry->{exp_tag}; + my $workdir = $chipEntry->{workdir}; + my $chip_imfile_id = $chipEntry->{chip_imfile_id}; + + # Process the image and burntool table + # IMAGE CHECK: Look up the image location, prove it exists, and if not, abort the prep for this object. + my ($ipp_uri, $remote_uri) = uri_convert($uri); + unless (-e $ipp_uri) { + # Try to fix it. + system("neb-repair $uri"); + ($ipp_uri, $remote_uri) = uri_convert($uri); + unless (-e $ipp_uri) { + # This image file is missing, so it will require manual intervention. + &my_die("Required image file $ipp_uri ($chip_id, $class_id) not found. Stopping this prep.", + $remote_id,$chip_id,$PS_EXIT_CONFIG_ERROR,$hard_fail_state); } } - } - -# -# Step 3: Iterate over the sub-components -# Iterate over the chipProcessedImfile level data. - my $reg_command = "$chiptool -pendingimfile "; - $reg_command .= " -chip_id $chip_id " if defined($chip_id); - $reg_command .= " -dbname $dbname " if defined($dbname); - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $reg_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - - &my_die("Unable to run chiptool -pendingimfile", - $chip_id,$error_code); - } - - my $chipData = $mdcParser->parse(join "", @$stdout_buf) or - next; -# &my_die("Unable to parse chiptool -pendingimfile information.", -# $chip_id,$error_code); - my $chipData2= parse_md_list($chipData); - - - foreach my $chipEntry (@$chipData2) { - # Get information we need to pass to ppImage - my $uri = $chipEntry->{uri}; - my $class_id = $chipEntry->{class_id}; - my $video_cells = $chipEntry->{video_cells}; - my $reduction = $chipEntry->{reduction}; - my $exp_tag = $chipEntry->{exp_tag}; - my $workdir = $chipEntry->{workdir}; - my $chip_imfile_id = $chipEntry->{chip_imfile_id}; - - # Process the image and burntool table - my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri); - my $btt = $uri; - $btt =~ s/fits$/burn.tbl/; - my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt); - - # Initialize the ppI command - my $ppImage_command = "ppImage -file $remote_uri"; - $ppImage_command .= " -burntool $remote_btt "; - - # Add detrend information to the command line - foreach my $det (keys %detrends_to_use) { - if ((($video_cells)&&($det eq 'DARK'))|| - ((!$video_cells)&&($det eq 'VIDEODARK'))) { - next; + + # The image exists, so continue. This resets the variables, but also outputs the file locations. + ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri); + my $btt = $uri; + $btt =~ s/fits$/burn.tbl/; + + # Check burntool table for existance, and if it doesn't, regenerate it. + my ($ipp_btt, $remote_btt) = uri_convert($btt); + unless (-e $ipp_btt) { + my $exp_name = $chipEntry->{exp_name}; + my $burntool_command = "$ipp_burntool_fix --exp_name $exp_name --class_id $class_id"; + $burntool_command .= " -dbname $dbname" if defined($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $burntool_command, verbose => $dbverbose); + # That command repairs the tables, but the instance we learned about may not be correct. + # Look up the new one. + ($ipp_btt, $remote_btt) = uri_convert($btt); + + unless (-e $ipp_btt) { + &my_die("Attempted regeneration of burntool table ($chip_id, $class_id) has failed. Stopping this prep.", + $remote_id,$chip_id, $PS_EXIT_CONFIG_ERROR, $hard_fail_state); } - my $det_code = $det_types{$det}; - my $duri = $detrends_to_use{$det}->{$class_id}; - my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); - $ppImage_command .= " $det_code $remote_det_uri "; } - - - # Add output root - my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; - my $remote_outroot = uri_local_to_remote($ipp_outroot); - $ppImage_command .= " $remote_outroot "; - print STDERR "$remote_outroot $ipp_outroot $class_id\n"; - # Complete reduction information. - $reduction = 'DEFAULT' unless defined $reduction; - my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage - my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot - - $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; - $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; - $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; - $ppImage_command .= " -threads 4 "; - $ppImage_command .= " -image_id $chip_imfile_id "; - $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; - $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; - -# push @main_commands, $ppImage_command; - - # Calculate pre and post commands - my $remote_outroot_dir = dirname($remote_outroot); - my $pre_command = "mkdir -p $remote_outroot_dir"; - - my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; - $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; - $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; - $post_commandA .= " -dbname $dbname " if defined $dbname; - - my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; - my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; - -# push @post_commands, "$post_commandB $post_commandC"; - - $job_index++; - - print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; - - - # Determine which output files need to be returned. - foreach my $component (@return_component_list) { -# uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) ); - my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); - my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); - my $remote_outroot_dir = dirname($ipp_disk); - print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; -# print " $filename $ipp_disk $remote_disk\n"; -# die; - } - - print CONFIG "\n"; - } -} - -# Actually put the commands into the output file. -# my %unique_cmds = (); -# foreach my $cmd (@pre_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@main_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } -# foreach my $cmd (@post_commands) { -# unless(exists($unique_cmds{$cmd})) { -# print CONFIG $cmd . "\n"; -# $unique_cmds{$cmd} = 1; -# } -# } + ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt); + + # Initialize the ppI command + my $ppImage_command = "ppImage -file $remote_uri"; + $ppImage_command .= " -burntool $remote_btt "; + + foreach my $det_type (keys (%det_types)) { + if (( $video_cells) && ($det_type eq 'DARK')) { next; } + if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; } + + my $det_id = $det_ids{$det_type}; + my $det_iter = $det_iters{$det_type}; + + if (not defined $det_id) { next; } + + if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; } + + # Add detrend information to the command line + my $detselect_command = "detselect -select "; + $detselect_command .= " -det_id $det_id"; + $detselect_command .= " -iteration $det_iter"; + $detselect_command .= " -class_id $class_id"; + $detselect_command .= " -dbname $dbname " if defined($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command, verbose => $dbverbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state); + } + + my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state); + my $detImfileMD = parse_md_list($detMDlist); + my $detImfile = $detImfileMD->[0]; + my $duri = $detImfile->{uri}; + + my $det_code = $det_types{$det_type}; + my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri); + $ppImage_command .= " $det_code $remote_det_uri "; + } + + # Add output root + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + $ppImage_command .= " $remote_outroot "; + print STDERR "$remote_outroot $ipp_outroot $class_id\n"; + # Complete reduction information. + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage + my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot + + $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; + $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; + $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; + $ppImage_command .= " -threads $threads_req "; + $ppImage_command .= " -image_id $chip_imfile_id "; + $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; + $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; + + # Calculate pre and post commands + my $remote_outroot_dir = dirname($remote_outroot); + my $pre_command = "mkdir -p $remote_outroot_dir"; + + my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; + $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; + $post_commandA .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 "; + $post_commandA .= " -dbname $dbname " if defined $dbname; + + my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo "; + my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; + + $job_index++; + + print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}"; + + # Determine which output files need to be returned. + foreach my $component (@return_component_list) { + my $filename = $ipprc->filename($component, $ipp_outroot, $class_id); + my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename ); + my $remote_outroot_dir = dirname($ipp_disk); + print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk "; + } + print CONFIG "\n"; + } +} close(CONFIG); @@ -386,42 +326,4 @@ close(GENERATE); -# -# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. -my $proc_need = $job_index * $threads; # how many total processors do we need? -my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? -my $time_need = $job_index * $job_cost; # How many seconds will this take? - -my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill -my ($time_req,$node_req); -if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { - $time_req = $min_time; - $node_req = $min_nodes; -} -elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { - $time_req = $max_time; - $node_req = $max_nodes; - print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; -} -else { - $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; - $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; -} -$time_req += 1; # Safety addition. - -open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); -print COMMAND "#!/bin/tcsh\n"; -print COMMAND "##### Moab controll lines\n"; -print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES -print COMMAND "#MSUB -j oe\n"; -print COMMAND "#MSUB -V\n"; -print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; -print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; -#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; -print COMMAND "date\n"; -close(COMMAND); - - ## We're done here. The execution and handling are done elsewhere. # Quick review: @@ -429,15 +331,13 @@ # auth unless($no_update) { - my $command = "remotetool -updaterun -remote_id $remote_id "; - $command .= " -set_state pending "; + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $chip_id "; + $command .= " -set_jobs $job_index"; + $command .= " -set_path_base $path_base"; + $command .= " -set_state prep_done"; $command .= " -dbname $dbname " if defined $dbname; system($command); } - - - - - +exit (0); sub uri_convert { @@ -445,7 +345,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri ); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -461,7 +361,7 @@ my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1); my $remote_disk = $ipp_disk; - + unless(defined($ipp_disk)) { - my_die(); + &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state); } @@ -476,10 +376,7 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); - - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -489,9 +386,7 @@ my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); $remote_disk = $remote_raw . $ipp_disk; - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print TRANSFER "$ipp_disk\n"; - print CHECK "$remote_disk\n"; - } + + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -500,9 +395,6 @@ my $neb_uri = shift; my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri ); - unless (exists($file_filter{$neb_uri})) { - $file_filter{$neb_uri} = 1; - print RETURN "$ipp_disk\n"; - print GENERATE "$remote_disk\n"; - } + print RETURN "$ipp_disk\n"; + print GENERATE "$remote_disk\n"; return($ipp_disk,$remote_disk); } @@ -518,5 +410,5 @@ return($remote_uri); } - + sub uri_remote_to_local { # This needs to replace the remote root directory with the nebulous tag. @@ -524,10 +416,12 @@ $remote_uri =~ s%${remote_root}%%; my $local_uri = "neb:///" . $remote_uri; - + return($local_uri); } + sub my_die { my $msg = shift; - my $id = shift; + my $remote_id = shift; + my $stage_id = shift; my $exit_code = shift; my $exit_state = shift; @@ -536,12 +430,11 @@ carp($msg); - - if (defined $id and not $no_update) { - my $command = "remotetool -updaterun -remote_id $id"; - $command .= " -fault $exit_code " if defined $exit_code; - $command .= " -set_state $exit_state " if defined $exit_state; - $command .= " -dbname $dbname " if defined $dbname; - - system($command); + + if (defined $remote_id and defined $stage_id and not $no_update) { + my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id"; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); }