Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl
===================================================================
--- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl	(revision 37281)
+++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl	(revision 37282)
@@ -21,12 +21,13 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root = '/scratch3/watersc1/';
 my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
+
+# value for chip:
 my $threads       = 2;                      # How many threads are we going to use?
 my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
+
 my $proc_per_node = 24;                     # processors per node
 my $min_nodes     = 1;                      # smallest allocation to ask for
@@ -40,5 +41,4 @@
 # Look for programs we need
 my $missing_tools;
-my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
 my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1);
@@ -71,96 +71,23 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    
-    &my_die("Unable to run chiptool to determine stage parameters.",
-	    $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-	    $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1a: Load up the catalog of detrends 
-# Detrend concept holders
-my %detrends = ();
-my %det_types = ('MASK' => '-mask',
-		 'FLAT' => '-flat',
-		 'DARK' => '-dark',
-		 'VIDEODARK' => '-dark',
-		 'LINEARITY' => '-linearity',
-		 'FRINGE'    => '-fringe',
-		 'NOISEMAP'  => '-noisemap');
-
-foreach my $det_type (keys (%det_types)) {
-    my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type ";
-    $dt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $dt_cmd, verbose => $verbose);
+# STEP 1: Get a list of the components that make up this remoteRun
+my $compData;
+{
+    my $command = "$remotetool -listcomponent -remote_id $remote_id -state prep_done";
+    $command   .= " -dbname $dbname " if defined($dbname);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
     unless ($success) {
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("UNable to run detselect to determine detrend catalog",
-		$remote_id,$error_code);
-    }
-    my $detData = $mdcParser->parse(join "", @$stdout_buf) or
-	&my_die("Unable to determine detrend information.",
-		$remote_id,$PS_EXIT_PROG_ERROR);
-    my $detData2= parse_md_list($detData);
-
-    # Data structure for this:
-    # $detrend{NAME} = \@list_of_detselect_shows
-    # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element)
-    # $detrend{NAME}[0]{class_id} = uri
-
-    # At this point, I have the list of detselect shows, assign it
-    $detrends{$det_type} = $detData2;
-
-    for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) {
-	my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id};
-	my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration};
-
-	my $detselect_command2 = "detselect -select ";
-	$detselect_command2   .= " -det_id $det_id ";
-	$detselect_command2   .= " -iteration $det_iter ";
-	$detselect_command2   .= " -dbname $dbname " if defined($dbname);
-	( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    run(command => $detselect_command2, verbose => 0);
-	unless ($success) {
-	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	    
-	    &my_die("No valid detrend available for this image: $det_type $det_id $det_iter",
-		    $chip_id,$error_code);
-	}
 	
-	my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or 
-	    &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter",
-		    $chip_id,$error_code);
-	my $detImfile2= parse_md_list($detImfile);
-	foreach $detImfile (@$detImfile2) {
-	    my $class_id = $detImfile->{class_id};
-	    my $duri = $detImfile->{uri};
-	    
-	    ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri;
-	}
-    }
-}
-
-#print Dumper(%detrends);
-
-my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
-
-#
-# Step 1b: Open output files
+	&my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $error_code);
+    }
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+	&my_die("Unable to determine component information.", $remote_id,$PS_EXIT_PROG_ERROR);
+    $compData = parse_md_list($MDlist);
+}
+
+# STEP 2: Open master output files
 my $uri_command = $path_base . ".cmd";
 my $uri_transfer= $path_base . ".transfer";
@@ -177,211 +104,63 @@
 my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
-my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
+my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. 
 
 open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
-
-
-#
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $chip_id,$PS_EXIT_SYS_ERROR);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $chip_id,$PS_EXIT_SYS_ERROR);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $chip_id,$PS_EXIT_SYS_ERROR);
+
 # Step 2: Iterate over all componenets in this remote run.
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-
-foreach my $compEntry (@$compData2) {
-
-    my $chip_id = $compEntry->{stage_id};
-    my $exp_id ;
-# Get exposure level information from the chipRun we're working from.
-    my $command = "$chiptool -listrun -chip_id $chip_id ";
-    $command   .= " -dbname $dbname " if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => 0);
-    unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	
-	&my_die("Unable to run chiptool to determine stage parameters.",
-		$chip_id,$error_code);
-    }
-    
-# Parse chipRun level data to determine detrend information 
-    my %detrends_to_use = ();
-    {
-	my $chipData = $mdcParser->parse(join "", @$stdout_buf) or
-	    &my_die("Unable to determine chip component information.",
-		    $chip_id,$PS_EXIT_PROG_ERROR);
-	my $chipData2= parse_md_list($chipData);
-	my $chipEntry = ${ $chipData2 }[0];
-	
-	$exp_id = $chipEntry->{exp_id};    
-	my $filter = $chipEntry->{filter};
-	my $altfilt= $filter;
-	$altfilt =~ s/.00000//;
-	my $dateobs= $chipEntry->{dateobs};
-	my ($date, $time) = split /T/, $dateobs;
-	my ($year,$month,$day) = split /-/, $date;
-	my ($hour,$minute,$second) = split /:/, $time; #/;
-	my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second);
-
-	foreach my $det_type (%detrends) {
-	    if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
-
-	    foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options
-		if (defined($det_obj->{filter})) { 
-		    if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) {
-			next; # Skip detrends with a filter that we can't use.
-		    }
-		}
-		if (defined($det_obj->{time_begin})) {
-		    if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) {
-			next; # Skip detrends that come into effect after this exposure (d1 < d2)
-		    }
-		}
-		if (defined($det_obj->{time_end})) {
-		    if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) {
-			next; # Skip detrends that stop working before this exposure (d1 > d2)
-		    }
-		}
-		$detrends_to_use{$det_type} = $det_obj;
-		last;
-	    }
-	}
-    }
-
-#
-# Step 3: Iterate over the sub-components
-# Iterate over the chipProcessedImfile level data.
-    my $reg_command = "$chiptool -pendingimfile ";
-    $reg_command   .= " -chip_id $chip_id " if defined($chip_id);
-    $reg_command   .= " -dbname $dbname "   if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $reg_command, verbose => $verbose);
-    unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	
-	&my_die("Unable to run chiptool -pendingimfile",
-		$chip_id,$error_code);
-    }
-    
-    my $chipData = $mdcParser->parse(join "", @$stdout_buf) or 
-	next;
-# 	&my_die("Unable to parse chiptool -pendingimfile information.",
-# 		$chip_id,$error_code);
-    my $chipData2= parse_md_list($chipData);
-
-    
-    foreach my $chipEntry (@$chipData2) {
-	# Get information we need to pass to ppImage
-	my $uri            = $chipEntry->{uri};
-	my $class_id       = $chipEntry->{class_id};
-	my $video_cells    = $chipEntry->{video_cells};
-	my $reduction      = $chipEntry->{reduction};
-	my $exp_tag        = $chipEntry->{exp_tag};
-	my $workdir        = $chipEntry->{workdir};
-	my $chip_imfile_id = $chipEntry->{chip_imfile_id};
-	
-	# Process the image and burntool table
-	my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
-	my $btt = $uri;
-	$btt =~ s/fits$/burn.tbl/;
-	my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
-	
-	# Initialize the ppI command
-	my $ppImage_command = "ppImage -file $remote_uri";
-	$ppImage_command   .= " -burntool $remote_btt ";
-	
-	# Add detrend information to the command line
-	foreach my $det (keys %detrends_to_use) {
-	    if ((($video_cells)&&($det eq 'DARK'))||
-		((!$video_cells)&&($det eq 'VIDEODARK'))) {
-		next;
-	    }
-	    my $det_code = $det_types{$det};
-	    my $duri = $detrends_to_use{$det}->{$class_id};
-	    my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
-	    $ppImage_command .= " $det_code $remote_det_uri ";
-	}
-	    
-	
-	# Add output root
-	my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
-	my $remote_outroot = uri_local_to_remote($ipp_outroot);
-	$ppImage_command .= " $remote_outroot ";
-	print STDERR "$remote_outroot $ipp_outroot $class_id\n";
-	# Complete reduction information.
-	$reduction = 'DEFAULT' unless defined $reduction;
-	my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
-	my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
-	
-	$ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
-	$ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
-	$ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
-	$ppImage_command .= " -threads 4 ";
-	$ppImage_command .= " -image_id $chip_imfile_id ";
-	$ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
-	$ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
-
-#	push @main_commands, $ppImage_command;
-
-	# Calculate pre and post commands
-	my $remote_outroot_dir = dirname($remote_outroot);
-	my $pre_command =  "mkdir -p $remote_outroot_dir";
-	
-	my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
-	$post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
-	$post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
-	$post_commandA   .= " -dbname $dbname " if defined $dbname;
-
-	my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
-	my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
-	
-#	push @post_commands, "$post_commandB $post_commandC";
-
-	$job_index++;
-
-	print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
-
-
-	# Determine which output files need to be returned.
-	foreach my $component (@return_component_list) {
-#	    uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
-	    my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
-	    my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
-	    my $remote_outroot_dir = dirname($ipp_disk);
-	    print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-#	    print "  $filename $ipp_disk $remote_disk\n";
-#	    die;
-	}
-
-	print CONFIG "\n";
-    }
-}
-
-# Actually put the commands into the output file.
-# my %unique_cmds = ();
-# foreach my $cmd (@pre_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-# 	print CONFIG $cmd . "\n";
-# 	$unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@main_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-# 	print CONFIG $cmd . "\n";
-# 	$unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@post_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-# 	print CONFIG $cmd . "\n";
-# 	$unique_cmds{$cmd} = 1;
-#     }
-# }
+foreach my $compEntry (@$compData) {
+    my $line;
+
+    my $stage_id = $compEntry->{stage_id};
+
+    my $in_path_base = $compEntry->{path_base};
+
+    my $uri_transfer = $in_path_base . ".transfer";
+    my $disk_transfer= $ipprc->file_resolve($uri_transfer);
+    open(INPUT, "$disk_transfer");
+    while ($line = <INPUT>) {
+	print TRANSFER $line;
+    }
+    close (INPUT);
+
+    my $uri_check = $in_path_base . ".check";
+    my $disk_check= $ipprc->file_resolve($uri_check);
+    open(INPUT, "$disk_check");
+    while ($line = <INPUT>) {
+	print CHECK $line;
+    }
+    close (INPUT);
+
+    my $uri_config = $in_path_base . ".config";
+    my $disk_config= $ipprc->file_resolve($uri_config);
+    open(INPUT, "$disk_config");
+    while ($line = <INPUT>) {
+	print CONFIG $line;
+    }
+    close (INPUT);
+
+    my $uri_generate = $in_path_base . ".generate";
+    my $disk_generate= $ipprc->file_resolve($uri_generate);
+    open(INPUT, "$disk_generate");
+    while ($line = <INPUT>) {
+	print GENERATE $line;
+    }
+    close (INPUT);
+
+    my $uri_return = $in_path_base . ".return";
+    my $disk_return= $ipprc->file_resolve($uri_return);
+    open(INPUT, "$disk_return");
+    while ($line = <INPUT>) {
+	print RETURN $line;
+    }
+    close (INPUT);
+
+    $job_index += $compEntry->{jobs};
+}
 
 close(CONFIG);
@@ -391,5 +170,4 @@
 close(GENERATE);
 
-#
 # Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
 my $proc_need = $job_index * $threads;       # how many total processors do we need?
@@ -424,8 +202,6 @@
 print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
 print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
-#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n";
 print COMMAND "date\n";
 close(COMMAND);
-
 
 ## We're done here. The execution and handling are done elsewhere.
@@ -441,8 +217,5 @@
 }
 
-
-    
-
-
+exit (0);
 
 sub uri_convert {
@@ -462,74 +235,4 @@
 }
 
-sub uri_convert_and_create {
-    my $neb_uri = shift;
-    my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
-    my $remote_disk = $ipp_disk;
-    
-    unless(defined($ipp_disk)) {
-	my_die();
-    }
-
-    $remote_disk =~ s%^.*/%%;   # Remove nebulous path
-    $remote_disk =~ s%^\d+\.%%; # Remove ins_id
-    $remote_disk =~ s%:%/%g;    # Replace colons with directories
-    $remote_disk = "${remote_root}/${remote_disk}";
-    return($ipp_disk,$remote_disk);
-}
-
-sub uri_to_outputs {
-    my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print TRANSFER "$ipp_disk\n";
-	print CHECK    "$remote_disk\n";
-    }
-    return($ipp_disk,$remote_disk);
-}
-
-sub uri_to_outputs_raw {
-    my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    $remote_disk = $remote_raw . $ipp_disk;
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print TRANSFER "$ipp_disk\n";
-	print CHECK    "$remote_disk\n";
-    }
-    return($ipp_disk,$remote_disk);
-}
-
-sub uri_to_outputs_for_return {
-    my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print RETURN "$ipp_disk\n";
-	print GENERATE "$remote_disk\n";
-    }
-    return($ipp_disk,$remote_disk);
-}
-
-sub uri_local_to_remote {
-    # This needs to replace the nebulous tag with the remote root.
-    my $local_uri = shift;
-    $local_uri =~ s%^.*?/%%; # neb:/
-    $local_uri =~ s%^.*?/%%; # /
-    $local_uri =~ s%^.*?/%%; # @HOST@.0/
-    my $remote_uri = "${remote_root}/" . $local_uri;
-
-    return($remote_uri);
-}
- 
-sub uri_remote_to_local {
-    # This needs to replace the remote root directory with the nebulous tag.
-    my $remote_uri = shift;
-    $remote_uri =~ s%${remote_root}%%;
-    my $local_uri  = "neb:///" . $remote_uri;
-    
-    return($local_uri);
-}
 sub my_die {
     my $msg = shift;