Index: /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_prepare_chip.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_prepare_chip.pl (revision 36622) +++ /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_prepare_chip.pl (revision 36622) @@ -0,0 +1,479 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; +use Data::Dumper; +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; # Far side destination base location +my $threads = 2; # How many threads are we going to use? +my $job_cost = 150 / 60 / 60; # Estimate of how long a job runs, in hours. +my $proc_per_node = 24; # processors per node +my $min_nodes = 1; # smallest allocation to ask for +my $max_nodes = 1000; # largest allocation to ask for +my $min_time = 1; # shortest allocation to ask for +my $max_time = 8; # longest allocation to ask for + +# We need to ensure we only ever try to transfer a file once. +my %file_filter = (); + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Options +my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update); +GetOptions( + 'remote_id=s' => \$remote_id, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, + 'path_base=s' => \$path_base, + 'no_update' => \$no_update, + 'verbose' => \$verbose, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless + defined($remote_id) and + defined($camera) and + defined($path_base) and + defined($dbname); +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# +# Step 1: Get a list of the components that make up this remoteRun + +# SHould this call listrun to ensure we're in state new? +my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id"; +$rt_cmd .= " -dbname $dbname " if defined($dbname); + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $rt_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $remote_id,$error_code); +} +my $compData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine component information.", + $remote_id,$PS_EXIT_PROG_ERROR); +my $compData2= parse_md_list($compData); + +# +# Step 1a: Load up the catalog of detrends +# Detrend concept holders +my %detrends = (); +my %det_types = ('MASK' => '-mask', + 'FLAT' => '-flat', + 'DARK' => '-dark', + 'VIDEODARK' => '-dark', + 'LINEARITY' => '-linearity', + 'FRINGE' => '-fringe', + 'NOISEMAP' => '-noisemap'); + +foreach my $det_type (keys (%det_types)) { + my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type "; + $dt_cmd .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $dt_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("UNable to run detselect to determine detrend catalog", + $remote_id,$error_code); + } + my $detData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine detrend information.", + $remote_id,$PS_EXIT_PROG_ERROR); + my $detData2= parse_md_list($detData); + + # Data structure for this: + # $detrend{NAME} = \@list_of_detselect_shows + # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element) + # $detrend{NAME}[0]{class_id} = uri + + # At this point, I have the list of detselect shows, assign it + $detrends{$det_type} = $detData2; + + for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) { + my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id}; + my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration}; + + my $detselect_command2 = "detselect -select "; + $detselect_command2 .= " -det_id $det_id "; + $detselect_command2 .= " -iteration $det_iter "; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $detselect_command2, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + } + + my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", + $chip_id,$error_code); + my $detImfile2= parse_md_list($detImfile); + foreach $detImfile (@$detImfile2) { + my $class_id = $detImfile->{class_id}; + my $duri = $detImfile->{uri}; + + ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri; + } + } +} + +#print Dumper(%detrends); + + +# +# Step 1b: Open output files +my $uri_command = $path_base . ".cmd"; +my $uri_transfer= $path_base . ".transfer"; +my $uri_check = $path_base . ".check"; +my $uri_config = $path_base . ".config"; + +my $disk_command = $ipprc->file_resolve($uri_command,1); +my $disk_transfer= $ipprc->file_resolve($uri_transfer,1); +my $disk_check = $ipprc->file_resolve($uri_check,1); +my $disk_config = $ipprc->file_resolve($uri_config,1); + +my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it. + +open(TRANSFER, ">$disk_transfer") || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR); +open(CHECK, ">$disk_check") || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR); +open(CONFIG, ">$disk_config") || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR); + +# +# Step 2: Iterate over all componenets in this remote run. +my $job_index = 0; +my @pre_commands = (); +my @main_commands = (); +my @post_commands = (); + +foreach my $compEntry (@$compData2) { + + my $chip_id = $compEntry->{stage_id}; + my $exp_id ; +# Get exposure level information from the chipRun we're working from. + my $command = "$chiptool -listrun -chip_id $chip_id "; + $command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool to determine stage parameters.", + $chip_id,$error_code); + } + +# Parse chipRun level data to determine detrend information + my %detrends_to_use = (); + { + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to determine chip component information.", + $chip_id,$PS_EXIT_PROG_ERROR); + my $chipData2= parse_md_list($chipData); + my $chipEntry = ${ $chipData2 }[0]; + + $exp_id = $chipEntry->{exp_id}; + my $filter = $chipEntry->{filter}; + my $altfilt= $filter; + $altfilt =~ s/.00000//; + my $dateobs= $chipEntry->{dateobs}; + my ($date, $time) = split /T/, $dateobs; + my ($year,$month,$day) = split /-/, $date; + my ($hour,$minute,$second) = split /:/, $time; #/; + my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second); + + foreach my $det_type (%detrends) { + if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y + + foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options + if (defined($det_obj->{filter})) { + if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) { + next; # Skip detrends with a filter that we can't use. + } + } + if (defined($det_obj->{time_begin})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) { + next; # Skip detrends that come into effect after this exposure (d1 < d2) + } + } + if (defined($det_obj->{time_end})) { + if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) { + next; # Skip detrends that stop working before this exposure (d1 > d2) + } + } + $detrends_to_use{$det_type} = $det_obj; + last; + } + } + } + +# +# Step 3: Iterate over the sub-components +# Iterate over the chipProcessedImfile level data. + my $reg_command = "$chiptool -pendingimfile "; + $reg_command .= " -chip_id $chip_id " if defined($chip_id); + $reg_command .= " -dbname $dbname " if defined($dbname); + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $reg_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + + &my_die("Unable to run chiptool -pendingimfile", + $chip_id,$error_code); + } + + my $chipData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse chiptool -pendingimfile information.", + $chip_id,$error_code); + my $chipData2= parse_md_list($chipData); + + + foreach my $chipEntry (@$chipData2) { + # Get information we need to pass to ppImage + my $uri = $chipEntry->{uri}; + my $class_id = $chipEntry->{class_id}; + my $video_cells = $chipEntry->{video_cells}; + my $reduction = $chipEntry->{reduction}; + my $exp_tag = $chipEntry->{exp_tag}; + my $workdir = $chipEntry->{workdir}; + my $chip_imfile_id = $chipEntry->{chip_imfile_id}; + + # Process the image and burntool table + my (undef,$remote_uri) = uri_to_outputs($uri); + my $btt = $uri; + $btt =~ s/fits$/burn.tbl/; + my (undef,$remote_btt) = uri_to_outputs($btt); + + # Initialize the ppI command + my $ppImage_command = "ppImage -file $remote_uri"; + $ppImage_command .= " -burntool $remote_btt "; + + # Add detrend information to the command line + foreach my $det (keys %detrends_to_use) { + if ((($video_cells)&&($det eq 'DARK'))|| + ((!$video_cells)&&($det eq 'VIDEODARK'))) { + next; + } + my $det_code = $det_types{$det}; + my $duri = $detrends_to_use{$det}->{$class_id}; + my (undef, $remote_det_uri) = uri_to_outputs($duri); + $ppImage_command .= " $det_code $remote_det_uri "; + } + + + # Add output root + my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}"; + my $remote_outroot = uri_local_to_remote($ipp_outroot); + $ppImage_command .= " $remote_outroot "; + print STDERR "$remote_outroot $ipp_outroot $class_id\n"; + # Complete reduction information. + $reduction = 'DEFAULT' unless defined $reduction; + my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage + my $recipe_psphot = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot + + $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage "; + $ppImage_command .= " -recipe PSPHOT $recipe_psphot "; + $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats "; + $ppImage_command .= " -threads $threads "; + $ppImage_command .= " -image_id $chip_imfile_id "; + $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace "; + $ppImage_command .= " -log ${remote_outroot}.${class_id}.log "; + + push @main_commands, $ppImage_command; + + # Calculate pre and post commands + my $remote_outroot_dir = dirname($remote_outroot); + push @pre_commands, "mkdir -p $remote_outroot_dir"; + + my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id "; + $post_commandA .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot "; + $post_commandA .= " -magicked 0 -hostname REMOTE -dtime_script 0 "; + $post_commandA .= " -dbname $dbname " if defined $dbname; + + my $post_commandB = "echo -n $post_commandA > ${remote_outroot}.${class_id}.dbinfo && "; + my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo"; + + push @post_commands, "$post_commandB $post_commandC"; + + $job_index++; + } +} + +# Actually put the commands into the output file. +my %unique_cmds = (); +foreach my $cmd (@pre_commands) { + unless(exists($unique_cmds{$cmd})) { + print CONFIG $cmd . "\n"; + $unique_cmds{$cmd} = 1; + } +} +foreach my $cmd (@main_commands) { + unless(exists($unique_cmds{$cmd})) { + print CONFIG $cmd . "\n"; + $unique_cmds{$cmd} = 1; + } +} +foreach my $cmd (@post_commands) { + unless(exists($unique_cmds{$cmd})) { + print CONFIG $cmd . "\n"; + $unique_cmds{$cmd} = 1; + } +} + +close(CONFIG); +close(TRANSFER); +close(CHECK); + +# +# Construct the moab command last, so we can use the job_index counter to estimate resources. Somehow. +my $proc_need = $job_index * $threads; # how many total processors do we need? +my $node_need = $proc_need / $proc_per_node; # this equals how many nodes? +my $time_need = $job_index * $job_cost; # How many seconds will this take? + +my $fill_factor = 0.8; # This is the factor of how much of the time allocation we'd like to fill +my ($time_req,$node_req); +if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) { + $time_req = $min_time; + $node_req = $min_nodes; +} +elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) { + $time_req = $max_time; + $node_req = $max_nodes; + print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job. This exceeds the max limits ($max_nodes, $max_time). Using those max values instead. Good luck.\n"; +} +else { + $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1; + $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1; +} + +open(COMMAND, ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR); +print COMMAND "#!/bin/tcsh\n"; +print COMMAND "##### Moab controll lines\n"; +print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES +print COMMAND "#MSUB -j oe\n"; +print COMMAND "#MSUB -V\n"; +print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n"; +print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n"; +#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n"; +print COMMAND "date\n"; +close(COMMAND); + + +## We're done here. The execution and handling are done elsewhere. +# Quick review: +# new -> pending -> run -> full +# auth +unless($no_update) { + my $command = "remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state pending "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); +} + + + + + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + + unless (exists($file_filter{$neb_uri})) { + $file_filter{$neb_uri} = 1; + print TRANSFER "$ipp_disk\n"; + print CHECK "$remote_disk\n"; + } + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "remotetool -updaterun -stage_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} + Index: /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_remote_exec.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_remote_exec.pl (revision 36622) +++ /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_remote_exec.pl (revision 36622) @@ -0,0 +1,385 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Hard coded values +my $DMZ_HOST = 'wtrw'; +my $SEC_HOST = 'mu-fe'; +my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/'; +my $remote_root = '/scratch3/watersc1/'; + +# tools +my $missing_tools; +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $scp = can_run('scp') or (warn "Can't find scp" and $missing_tools = 1); +my $remotetool = can_run('echo') or (warn "Can't find remotetool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Options +my ($remote_id,$path_base,$policy,$poll,$job_id,$dbname,$verbose,$no_update,$camera); + +GetOptions( + 'remote_id=s' => \$remote_id, + 'job_id=s' => \$job_id, + 'path_base=s' => \$path_base, + 'policy=s' => \$policy, + 'poll' => \$poll, + 'camera=s' => \$camera, + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'no_update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless + defined($path_base) and + defined($remote_id); + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Phase 1: See if we can actually do anything. +# If the link is down, there's no benefit in trying to do anything else. +&check_ssh_connection(); + +print "passed authentication challenge.\n"; + +# These are operations we only need to do on the first call, not on subsequent poll operations. +unless ($poll) { +# Phase 2: Ensure files are in place: +# Copy command files + my @files = (); + + my $uri_command = $path_base . ".cmd"; + my $uri_transfer= $path_base . ".transfer"; + my $uri_check = $path_base . ".check"; + my $uri_config = $path_base . ".config"; + + my $disk_command = $ipprc->file_resolve($uri_command); + my $disk_transfer= $ipprc->file_resolve($uri_transfer); + my $disk_check = $ipprc->file_resolve($uri_check); + my $disk_config = $ipprc->file_resolve($uri_config); + + scp_put($disk_command,uri_local_to_remote($uri_command)); + scp_put($disk_transfer,uri_local_to_remote($uri_transfer)); + scp_put($disk_check,uri_local_to_remote($uri_check)); + scp_put($disk_config,uri_local_to_remote($uri_config)); + +# Run check command + my (undef,$remote_check) = uri_convert($uri_check); + my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check"); + +# Parse stdout for files to transfer. + if ($#{ $ssh_check_stdout } != -1) { # There were files not found. + # Someone was missing. Use the check and transfer lists to identify the local version, and send it. + # What was missing? + my %missing_files = (); + foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) { + print "Missing!: $l\n"; + $missing_files{$l} = 1; + } + + # Do we have that file? + my @check_files = (); + open(CHECK,$disk_check); + while() { + chomp; + push @check_files, $_; + } + close(CHECK); + + my @trans_files = (); + open(TRANS,$disk_transfer); + while() { + chomp; + push @trans_files, $_; + } + close(TRANS); + + if ($#trans_files != $#check_files) { &my_die("The transfer file list and check file list are different sizes", + $remote_id,$PS_EXIT_DATA_ERROR); } + + for (my $i = 0; $i <= $#check_files; $i++) { + if (exists($missing_files{$check_files[$i]})) { + if ($trans_files[$i] ne "REMOTE_ONLY") { + scp_put($trans_files[$i],"$check_files[$i]"); + } + else { + &my_die("There is a file that should be there, but the file isn't there, and I don't have a copy of that file here. The file is: $check_files[$i]", + $remote_id,$PS_EXIT_DATA_ERROR); # This seems a bit harsh. + } + } + } + } + + +# Run real command + my (undef,$remote_command) = uri_convert($uri_command); + my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command"); + if ($#{ $ssh_exec_stdout } != -1) { # Parse the output + $job_id = ${ $ssh_exec_stdout }[0]; + chomp($job_id); + $job_id =~ s/\s+//g; + } + + unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. + &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); + } + + # Notify the database that this entry is currently running. + my $command = "$remotetool -updaterun -remote_id $remote_id "; + $command .= " -set_state run "; + $command .= " -set_job_id $job_id "; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + +} + +# Check that we have a valid job_id, either from the command line, or from the msub command. +unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly. + &my_die("No job_id returned. Sorry.",$remote_id,$PS_EXIT_PROG_ERROR); +} + + +# Poll the job status? +my $poll_count = 0; +my $poll_max = 5; +my $poll_sleep = 60; +my $poll_response = 0; +while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)) { + if ($verbose) { + print "Polling code. Sleeping. $poll_count of $poll_max $poll_response\n"; + } + unless ($poll_count == 0) { + sleep($poll_sleep); + } + $poll_response = poll_job($job_id); + + if ($poll_response == 1) { # This job has completed + last; + } + elsif ($poll_response == -1) { # This job has an error + &my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR); + } + $poll_count++; +} +if ($verbose) { + print "Stopped Polling code. $poll_count of $poll_max $poll_response\n"; +} +# Retrieve validation summary +my $fault = 0; + +exit(1); +# Read this as a metadata object +my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out"); +my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout); +my $retrieveData2= parse_md_list($retrieveData); + +foreach my $retrieveEntry (@$retrieveData2) { + my $process_fault = $retrieveEntry->{fault}; + my $dbcommand = $retrieveEntry->{dbcommand}; + my $filelist = $retrieveEntry->{filelist}; + + scp_get("${filelist}",${filelist}); + open(FL,${filelist}); + while() { + chomp; + my $f = $_; + scp_get("${f}",$f); + } + close(FL); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $dbcommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("The database command returned did not execute correctly. $dbcommand $process_fault", + $remote_id,$error_code); + } + + if ($process_fault != 0) { + $fault = 1; + } +} + +if (($policy ne 'ignore')&&($fault != 0)) { + &my_die("This job exited without all components finishing successfully. Halting.", + $remote_id,$fault,"halt"); +} +else { + print "This job exited without all components finishing successfully. $fault\n" if $fault; + &my_die("This job finished. Setting remote job to full.", + $remote_id,0,"full"); +} + +# Promote to finished. +&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders. +# Quick review: +# new -> pending -> run -> full +# auth + +# END PROGRAM + + +sub check_ssh_connection { + my $cmd = "$ssh -O check $DMZ_HOST"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Authorization check failed.",$remote_id,0,'auth'); + +# my $update_command = "$remotetool -updaterun -remote_id $remote_id "; +# $update_command .= " -set_state auth "; +# $update_command .= " -dbname $dbname " if defined $dbname; +# system($update_command); +# exit(0); # This isn't an error. It's a state we monitor and move to automatically. + } +} + +sub scp_put { + my $file = shift; + my $destination = shift; + my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}"; + + my $directory = dirname($destination); + ssh_exec_command("mkdir -p $directory"); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die(); + } +} + +sub scp_get { + my $destination = shift; + my $file = shift; + + my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file "; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die(); + } +} + +sub ssh_exec_command { + my $cmd = shift; + $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die(); + } + return ($stdout_buf); +} + +sub poll_job { + my $job_id = shift; + my $response_array = ssh_exec_command("checkjob -v $job_id"); + my $response = join "\n", @$response_array; + + if ($response =~ /State: Running/) { + return(0); + } + elsif ($response =~ /State: Completed/) { + return(1); + } + elsif ($response =~ /State: Idle/) { + return(-2); + } + else { + return(-1); + } +} + + +sub uri_convert { + my $neb_uri = shift; + my $ipp_disk= $ipprc->file_resolve( $neb_uri ); + my $remote_disk = $ipp_disk; + + unless(defined($ipp_disk)) { + my_die(); + } + + $remote_disk =~ s%^.*/%%; # Remove nebulous path + $remote_disk =~ s%^\d+\.%%; # Remove ins_id + $remote_disk =~ s%:%/%g; # Replace colons with directories + $remote_disk = "${remote_root}/${remote_disk}"; + return($ipp_disk,$remote_disk); +} + +sub uri_to_outputs { + my $neb_uri = shift; + my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri ); + +# print TRANSFER "$ipp_disk\n"; +# print CHECK "$remote_disk\n"; + return($ipp_disk,$remote_disk); +} + +sub uri_local_to_remote { + # This needs to replace the nebulous tag with the remote root. + my $local_uri = shift; + $local_uri =~ s%^.*?/%%; # neb:/ + $local_uri =~ s%^.*?/%%; # / + $local_uri =~ s%^.*?/%%; # @HOST@.0/ + my $remote_uri = "${remote_root}/" . $local_uri; + + return($remote_uri); +} + +sub uri_remote_to_local { + # This needs to replace the remote root directory with the nebulous tag. + my $remote_uri = shift; + $remote_uri =~ s%${remote_root}%%; + my $local_uri = "neb:///" . $remote_uri; + + return($local_uri); +} +sub my_die { + my $msg = shift; + my $id = shift; + my $exit_code = shift; + my $exit_state = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $id and not $no_update) { + my $command = "$remotetool -updaterun -stage_id $id"; + $command .= " -fault $exit_code " if defined $exit_code; + $command .= " -set_state $exit_state " if defined $exit_state; + $command .= " -dbname $dbname " if defined $dbname; + + system($command); + } + + exit($exit_code); +} Index: /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_files.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_files.pl (revision 36622) +++ /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_files.pl (revision 36622) @@ -0,0 +1,68 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; +my $host_iter = 0; +my @ipp_hosts = ('166.122.172.41','166.122.172.42','166.122.172.43','166.122.172.44', + '166.122.172.45','166.122.172.46','166.122.172.47','166.122.172.48', + '166.122.172.49','166.122.172.50','166.122.172.51','166.122.172.52','166.122.172.53'); +my @remote_hosts = ('mu-fe1.lanl.gov','mu-fe2.lanl.gov','mu-fe3.lanl.gov','mu-fe4.lanl.gov'); +my $check_list; +my $transfer_list; +GetOptions( + 'check_list=s' => \$check_list, + 'transfer_list=s' => \$transfer_list + ) or pod2usage( 2 ); + +my @check_files = (); +my @transfer_files = (); + +open(LIST,$check_list); +while() { + chomp; + push @check_files, $_; +} +close(LIST); +open(LIST2,$transfer_list); +while() { + chomp; + push @transfer_files, $_; +} +close(LIST2); + +unless ($#check_files == $#transfer_files) { + die "The two lists are of unequal length."; +} + +for (my $i = 0; $i <= $#check_files; $i++) { + my $file = $check_files[$i]; + unless (-e $file) { + print "$file\n"; + my $fetch_file = $transfer_files[$i]; + my $directory = dirname($file); + unless (-d $directory) { + system("mkdir -p $directory"); + } + + scp_pull($fetch_file,$file); + } +} + +sub scp_pull { + my $remote_file = shift; + my $local_file = shift; + + my $host = $ipp_hosts[$host_iter]; + $host_iter++; + if ($host_iter > $#ipp_hosts) { $host_iter = 0; } + system("scp ${host}:${remote_file} ${local_file}"); +} Index: /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_processing.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_processing.pl (revision 36622) +++ /branches/eam_branches/ipp-20140226/ippScripts/scripts/sc_validate_processing.pl (revision 36622) @@ -0,0 +1,79 @@ +#! /usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +# Hard coded values +my $remote_root = '/scratch3/watersc1/'; + +my $command_output +GetOptions( + 'command_output=s' => \$command_output; + ) or pod2usage( 2 ); + +open(LIST,$command_output); +while() { + chomp; + + if ($_ =~ /ppImage/) { + my $base = (split /\s+/)[15]; # This is super fragile. + my $stats= $base . "." . $class_id . ".stats"; # no good idea how to get that. + my $command = ""; + if (-e $stats) { + my $stat_cmd = "ppStatsFromMetadata $stats - CHIP_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + # Need to get all these parameters. + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -uri $outputImage"; + $command .= " -path_base $outroot"; + $command .= " -magicked $magicked" if $magicked; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + + } + else { # This job failed permanently + $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -uri $outputImage" if defined $outputImage; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + } + + +# if ($_ =~ /stask: error: .*: task .*: Exited with exit code/) { +# print " "; # This needs to be smarter. +# exit(1); +# } +} +close(LIST); + Index: /branches/eam_branches/ipp-20140226/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20140226/ippToPsps/jython/cleanup.py (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippToPsps/jython/cleanup.py (revision 36622) @@ -87,5 +87,5 @@ self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs)) self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs)) - + # self.logger.infoPair("Did I get here", "yes?") # delete stuff from local disk if self.skychunk.deleteLocal: @@ -110,11 +110,13 @@ self.reportResults("datastore", deleteFromDatastoreIDs, count) - + # self.logger.infoPair("here","here") # remove stuff from DXLayer if self.skychunk.deleteDxLayer: +# self.logger.infoPair ("skip remove from dxlayer", "HAF") count = 0 for id in deleteFromDxLayerIDs: - + self.logger.infoPair("delete from dxlayer", id); if self.dxlayer.deleteBatch(id): + self.logger.infoPair("delete dxlayer",id); self.ippToPspsDb.updateDeletedDxlayer(id, 1) count = count + 1 Index: /branches/eam_branches/ipp-20140226/ippToPsps/jython/dxlayer.py =================================================================== --- /branches/eam_branches/ipp-20140226/ippToPsps/jython/dxlayer.py (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippToPsps/jython/dxlayer.py (revision 36622) @@ -37,8 +37,8 @@ if p.returncode != 0: - self.logger.debugPair("DXLayer webservice", "failed") + self.logger.infoPair("DXLayer webservice", "failed") return 0 else: - self.logger.debugPair("DXLayer webservice", "successful") + self.logger.infoPair("DXLayer webservice", "successful") ret = self.decodeDXLayerResponse(tempFile.name) @@ -48,5 +48,5 @@ if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) - else: self.logger.errorPair("Unable to delete batch from DXLayer 1" , batch) + else: self.logger.infoPair("Unable to delete batch from DXLayer 1" , batch) @@ -61,5 +61,5 @@ if p.returncode != 0: - self.logger.debugPair("DXLayer webservice", "failed") + self.logger.infoPair("DXLayer webservice", "failed") return 0 else: @@ -72,5 +72,5 @@ if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) - else: self.logger.errorPair("Unable to delete batch from DXLayer 2" , batch) + else: self.logger.infoPair("Unable to delete batch from DXLayer 2" , batch) Index: /branches/eam_branches/ipp-20140226/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20140226/ippToPsps/jython/odm.py (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippToPsps/jython/odm.py (revision 36622) @@ -78,5 +78,5 @@ DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text results['DETAILS'] = DETAILS - elif re.search("processing", STATE): + elif re.search("processing", STATE): loadedToODM = 1 if re.search("MergeWorthy", MESSAGE): @@ -87,4 +87,12 @@ mergeWorthy = 1 merged = 1 +# if re.search("released", STATE): +# loadedToODM = 1 +# mergeWorthy = 1 +# merged = 1 +# if re.search("released", MESSAGE): +# loadedToODM = 1 +# mergeWorthy = 1 +# merged = 1 except: pass Index: /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer.pl (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer.pl (revision 36622) @@ -18,5 +18,10 @@ ); -my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + +#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxwebservice.cgi"; +my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxlayer/dxwebservice.cgi"; + + #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; Index: /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer2.pl =================================================================== --- /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer2.pl (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer2.pl (revision 36622) @@ -18,5 +18,8 @@ ); -my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; + +#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxwebservice.cgi"; +my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxlayer/dxwebservice.cgi"; #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; Index: /branches/eam_branches/ipp-20140226/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20140226/ippTools/src/difftool.c (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippTools/src/difftool.c (revision 36622) @@ -1349,4 +1349,12 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psString warpDataGroup = psMetadataLookupStr(&mdok, row, "warpDataGroup"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warpDataGroup not found"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1391,5 +1399,5 @@ workdir, label, - data_group ? data_group : label, + data_group ? data_group : warpDataGroup, dist_group, reduction, Index: /branches/eam_branches/ipp-20140226/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20140226/ippconfig/recipes/filerules-split.mdc (revision 36621) +++ /branches/eam_branches/ipp-20140226/ippconfig/recipes/filerules-split.mdc (revision 36622) @@ -303,5 +303,5 @@ SKYCELL.TEMPLATE OUTPUT {OUTPUT}.skycell SKYCELL NONE FPA TRUE NONE -PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSUB.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE PPSUB.OUTPUT.MASK OUTPUT {OUTPUT}.mask.fits MASK COMP_MASK FPA TRUE NONE PPSUB.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE @@ -311,5 +311,5 @@ PPSUB.OUTPUT.JPEG2 OUTPUT {OUTPUT}.b2.jpg JPEG NONE FPA TRUE NONE PPSUB.OUTPUT.RESID.JPEG OUTPUT {OUTPUT}.res.jpg JPEG NONE FPA TRUE NONE -PPSUB.INVERSE OUTPUT {OUTPUT}.inv.fits IMAGE COMP_SUB FPA TRUE NONE +PPSUB.INVERSE OUTPUT {OUTPUT}.inv.fits IMAGE NONE FPA TRUE NONE PPSUB.INVERSE.MASK OUTPUT {OUTPUT}.inv.mask.fits MASK COMP_MASK FPA TRUE NONE PPSUB.INVERSE.VARIANCE OUTPUT {OUTPUT}.inv.wt.fits VARIANCE COMP_WT FPA TRUE NONE Index: /branches/eam_branches/ipp-20140226/ppBackground/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/Makefile.am (revision 36621) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/Makefile.am (revision 36622) @@ -1,3 +1,3 @@ -bin_PROGRAMS = ppBackground +bin_PROGRAMS = ppBackground ppBackgroundStack if HAVE_SVNVERSION @@ -28,4 +28,7 @@ ppBackground_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PPBACKGROUND_LIBS) +ppBackgroundStack_CPPFLAGS = $(ppBackground_CPPFLAGS) +ppBackgroundStack_LDFLAGS = $(ppBackground_LDFLAGS) + ppBackground_SOURCES = \ ppBackground.c \ @@ -38,4 +41,16 @@ ppBackgroundErrorCodes.c \ ppBackgroundExit.c + +ppBackgroundStack_SOURCES = \ + ppBackgroundStack.c \ + ppBackgroundStackArguments.c \ + ppBackgroundStackCamera.c \ + ppBackgroundStackData.c \ + ppBackgroundStackLoop.c \ + ppBackgroundStackMath.c \ + ppBackgroundVersion.c \ + ppBackgroundErrorCodes.c \ + ppBackgroundExit.c + noinst_HEADERS = \ Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackground.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackground.c (revision 36621) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackground.c (revision 36622) @@ -18,17 +18,19 @@ ppBackgroundErrorRegister(); + printf("Initializing data\n"); ppBackgroundData *data = ppBackgroundDataInit(&argc, argv); if (!data) { goto DIE; } - + printf("Reading Arguments\n"); if (!ppBackgroundArguments(data, argc, argv)) { goto DIE; } - + printf("Setting up Camera\n"); if (!ppBackgroundCamera(data)) { goto DIE; } + printf("Looping over data!\n"); if (!ppBackgroundLoop(data)) { goto DIE; Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.c (revision 36622) @@ -0,0 +1,75 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppBackgroundStack.h" + +int main(int argc, char *argv[]) +{ + ppBackgroundVersionPrint(); + + pmErrorRegister(); + psphotErrorRegister(); + ppBackgroundErrorRegister(); + + printf("Data init\n"); + ppBackgroundStackData *data = ppBackgroundStackDataInit(&argc, argv); + if (!data) { + goto DIE; + } + printf("Read arguments\n"); + if (!ppBackgroundStackArguments(data, argc, argv)) { + goto DIE; + } + + printf("Setup camera\n"); + if (!ppBackgroundStackCamera(data)) { + goto DIE; + } + + printf("Do Loop\n"); + if (!ppBackgroundStackLoop(data)) { + goto DIE; + } + + DIE: + ; // Empty statement to satisy compiler + psExit exitValue = ppBackgroundExitCode(PS_EXIT_SUCCESS); // Exit code + +/* if (data && data->stats && data->statsFile) { */ +/* psString stats = psMetadataConfigFormat(data->stats); // Statistics to output */ +/* if (!stats || strlen(stats) == 0) { */ +/* psError(PPBACKGROUND_ERR_IO, false, "Unable to format statistics file"); */ +/* } else if (fprintf(data->statsFile, "%s", stats) != strlen(stats)) { */ +/* psError(PPBACKGROUND_ERR_IO, true, "Unable to write statistics file"); */ +/* } */ +/* psFree(stats); */ +/* if (fclose(data->statsFile) == EOF) { */ +/* psError(PPBACKGROUND_ERR_IO, true, "Unable to close statistics file"); */ +/* } */ +/* data->statsFile = NULL; */ +/* exitValue = ppBackgroundExitCode(exitValue); */ +/* } */ + + if (data) { + psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "-dumpconfig"); + if (dump_file) { + if (!pmConfigDump(data->config, dump_file)) { + psError(psErrorCodeLast(), false, "Unable to dump configuration."); + exitValue = ppBackgroundExitCode(exitValue); + } + } + psFree(data); + } + + pmConfigDone(); + psLibFinalize(); + + return ppBackgroundExitCode(exitValue); +} + Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.h =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.h (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStack.h (revision 36622) @@ -0,0 +1,72 @@ +#ifndef PP_BACKGROUNDSTACK_H +#define PP_BACKGROUNDSTACK_H + +#include +#include + +#include "ppBackgroundErrorCodes.h" + +#define PPBACKGROUND_RECIPE "PPBACKGROUND_STACK" // Recipe name + +// Data for processing +typedef struct { + psMetadata *contents; // Metadata containing information about the data arrays + psArray *smfs; // List of pointers to individual smf headers + + // data->models->name->XY__->{image/ra/dec/calibrated/offset/scale} + // data->models->counter->XY__->{image/ra/dec/calibrated/offset/scale} + psMetadata *models; // Metadata containing the information about the model data + + psMetadata *OTA_solutions; // List of pointers to the OTA-specific solution images + bool fit_OTAS; // Calculate OTA solutions based on these inputs. + psString OTApath; // Location for pre-solved OTA solutions. + + psImageMap *modelMap; + psF32 ra_min; + psF32 ra_max; + psF32 dec_min; + psF32 dec_max; + + psArray *stack_data; + psArray *stacks; // List of stacks to be corrected. + psString outRoot; // Output root name + pmConfig *config; // Configuration +} ppBackgroundStackData; + +/// Initialise data for processing +ppBackgroundStackData *ppBackgroundStackDataInit(int *argc, char *argv[] // Command-line arguments + ); + +/// Parse command-line arguments +bool ppBackgroundStackArguments(ppBackgroundStackData *data, // Data for processing + int argc, char *argv[] // Command-line arguments + ); + +/// Parse camera configurations +bool ppBackgroundStackCamera(ppBackgroundStackData *data // Data for processing + ); + +/// Loop over input data, processing +bool ppBackgroundStackLoop(ppBackgroundStackData *data // Data for processing + ); + +bool ppBackgroundStackModelFitOTASolution(ppBackgroundStackData *data); +bool ppBackgroundStackDataModelFit(ppBackgroundStackData *data); +bool ppBackgroundStackCalibApply(ppBackgroundStackData *data); +bool ppBackgroundStackModelFit(ppBackgroundStackData *data); + + +/// Determine exit code +psExit ppBackgroundExitCode( + psExit exitValue // Current exit code + ); + +/// Add version information to header +bool ppBackgroundVersionHeader( + psMetadata *header // Header to supplement + ); + +/// Print version information to stdout +void ppBackgroundVersionPrint(void); + +#endif Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackArguments.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackArguments.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackArguments.c (revision 36622) @@ -0,0 +1,91 @@ +/** @file ppBackgroundStackArguments.c + * + * @brief + * + * @ingroup ppBackgroundStack + * + * @author Paul Price + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppBackgroundStack.h" + +/// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + ppBackgroundStackData *data // Run-time data + ) +{ + fprintf(stderr, "\nPan-STARRS background replacement\n\n"); + fprintf(stderr, "Usage: %s OUTPUT_ROOT\n\n", program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + psFree(data); + + pmConfigDone(); + psLibFinalize(); + + exit(PS_EXIT_CONFIG_ERROR); +} + + +bool ppBackgroundStackArguments(ppBackgroundStackData *data, int argc, char *argv[]) +{ + assert(data); + assert(data->config); + + psMetadata *arguments = data->config->arguments; + psMetadataAddStr(arguments, PS_LIST_TAIL, "-input", 0, "Filename of input metadata", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-image", 0, "Filename of image (required)", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask", 0, "Filename of mask", NULL); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-fitOTAs", 0, "", false); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-OTApath", 0, "", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-dumpconfig", 0, "", NULL); + + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 2) { + usage(argv[0], arguments, data); + } + + unsigned int numBad = 0; // Number of bad lines + data->contents = psMetadataConfigRead(NULL, &numBad,psMetadataLookupStr(NULL, arguments, "-input"),false); + if (!(data->contents) || numBad > 0) { + psError(PPBACKGROUND_ERR_CONFIG, false, "Unable to cleanly read MDC file with inputs."); + return(false); + } + + // Add any images to generate for to the list + // If we're going to do this for a full projection cell, this probably needs to be a metadata object. + psArrayAdd(data->stacks, data->stacks->n, psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-image"))); + + // Determine what we're doing with the OTA solution + if ((!psMetadataLookupBool(NULL,arguments,"-fitOTAs"))&& + (!psMetadataLookupStr(NULL,arguments,"-OTApath"))) { + psError(PPBACKGROUND_ERR_CONFIG, false, "No OTA solution path (-OTApath) provided, and no request to model this (-fitOTAs)."); + return(false); + } + data->fit_OTAS = psMetadataLookupBool(NULL, arguments, "-fitOTAs"); + data->OTApath = psMetadataLookupStr(NULL, arguments, "-OTApath"); + + + // This is the output base + data->outRoot = psStringCopy(argv[1]); + psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Output root name", data->outRoot); + + psTrace("ppBackgroundStack", 1, "Done reading command-line arguments\n"); + + + PS_ASSERT_STRING_NON_EMPTY(data->outRoot, false); + + return true; +} + + Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackCamera.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackCamera.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackCamera.c (revision 36622) @@ -0,0 +1,339 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppBackgroundStack.h" + +/// Add a single filename to the arguments as an array, so that it can be used with pmFPAfileBindFromArgs, etc +static void fileArguments(const char *file, // The symbolic name for the file + const char *name, // The name of the file + const char *comment, // Description of the file + pmConfig *config // Configuration + ) +{ + psArray *files = psArrayAlloc(1); // Array with file names + files->data[0] = psStringCopy(name); + if (psMetadataLookup(config->arguments, file)) { + psMetadataRemoveKey(config->arguments, file); + } + psMetadataAddArray(config->arguments, PS_LIST_TAIL, file, 0, comment, files); + psFree(files); + return; +} + + +bool ppBackgroundStackCamera(ppBackgroundStackData *data // Run-time data + ) +{ + bool status; // Status of file definition + pmConfig *config = data->config; // Because I'm reusing code. + int u,v; + size_t A,P; + double RR = -9999.0 ,DD = -99999.0,rr = 9999.0,dd = 99999.0; + // FIX Figure out what stacks we have to deal with. + psString stackName = data->stacks->data[0]; + fileArguments("IMAGE", stackName, "Input image", data->config); + psFree(stackName); + pmFPAfile *stack = pmFPAfileDefineFromArgs(&status, data->config, + "PPBACKGROUND.STACK", "IMAGE"); + if (!status || !stack) { + psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.STACK"); + return false; + } + + if (!pmAstromReadBilevelMosaic(stack->fpa,stack->fpa->hdu->header)) { + psError(psErrorCodeLast(), false, "Unable to read WCS astrometry for input stack."); + return false; + } + psArrayAdd(data->stack_data,data->stack_data->n,psMemIncrRefCounter(stack)); + pmFPAfileActivate(config->files, false, NULL); + + // You know what? Let's just fucking lie to config->files. + psMetadata *config_files = config->files; + config->files = NULL; + config->files = psMetadataAlloc(); + + // Read over the input background models. + psMetadataIterator *iter = psMetadataIteratorAlloc(data->contents, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + int i = 0; + while ((item = psMetadataGetAndIncrement(iter))) { + i++; + if (item->type != PS_DATA_METADATA) { + psError(PPBACKGROUND_ERR_ARGUMENTS, true, + "Component %s of the input metadata is not of type METADATA", item->name); + psFree(iter); + return(false); + } + + // Pull out the information for this exposure + psMetadata *input = item->data.md; // the input metadata of interest + psString smfFileName = psMetadataLookupStr(NULL, input, "astrom"); + psMetadata *modelContent = psMetadataLookupMetadata(NULL, input, "models"); + + // Allocate the model metadata object + psMetadata *Bmodel = psMetadataAlloc(); + + // Read the smf file from this item + fileArguments("astrom",smfFileName,"",config); + pmFPAfile *smfFile = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM","astrom"); + + // taking from pswarpLoadAstrometry.c + smfFile->type = PM_FPA_FILE_WCS; + // Read the SMF data + pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return NULL; + } + if (!pmFPAfileRead(smfFile,view,config)) { + psError(PS_ERR_IO, false, "failed READ at FPA %s", smfFile->name); + psFree(view); + return false; + } + + printf("CZW: Item %d\n",i); + // find the FPA phu + bool bilevelAstrometry = false; + pmHDU *phu = pmFPAviewThisPHU(view, smfFile->fpa); + if (phu) { + char *ctype = psMetadataLookupStr(NULL, phu->header, "CTYPE1"); + if (ctype) { + bilevelAstrometry = !strcmp (&ctype[4], "-DIS"); + } + } + if (bilevelAstrometry) { + if (!pmAstromReadBilevelMosaic(smfFile->fpa, phu->header)) { + psError(psErrorCodeLast(), false, "Unable to read bilevel mosaic astrometry for input FPA."); + psFree(view); + return false; + } + } + + psMemStats(0,&A,&P); + fprintf(stderr,"fpa %ld %ld\n",A,P); + + pmChip *chip; // Chip from FPA + while ((chip = pmFPAviewNextChip(view, smfFile->fpa, 1))) { + if (!chip->process || !chip->file_exists) { + continue; + } + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); // Name of chip + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(psErrorCodeLast(), false, "Error loading data from files."); + return false; + } + if (chip->cells->n != 1) { + psWarning("More than one cell present for chip %d", view->chip); + } + + psMemStats(0,&A,&P); + fprintf(stderr,"chip %ld %ld\n",A,P); + + // read WCS data from the corresponding header + pmHDU *hdu = pmFPAviewThisHDU (view, smfFile->fpa); + if (bilevelAstrometry) { + if (!pmAstromReadBilevelChip (chip, hdu->header)) { + psWarning("Unable to read bilevel chip astrometry for chip %s.", chipName); + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(psErrorCodeLast(), false, "Error saving data to files."); + return false; + } + continue; + } + } else { + // we use a default FPA pixel scale of 1.0 + psWarning("Reading WCS astrometry for chip %s.", chipName); + if (!pmAstromReadWCS(smfFile->fpa, chip, hdu->header, 1.0)) { + psError(psErrorCodeLast(), false, "Unable to read WCS astrometry for input FPA."); + psFree(view); + return false; + } + } + + // Load model data for this chip + psString modelFileName = pmConfigConvertFilename(psMetadataLookupStr(NULL, modelContent, chipName), + config, PM_FPA_MODE_READ, false); + psFits *modelFits = psFitsOpen(modelFileName,"r"); + psImage *image = psFitsReadImage(modelFits,psRegionSet(0,0,0,0),0); + psMetadata *header = psFitsReadHeader(NULL,modelFits); + + // Allocate the data structures for this chip + psImage *raim = psImageAlloc(image->numCols,image->numRows,PS_TYPE_F32); + psImage *decim = psImageAlloc(image->numCols,image->numRows,PS_TYPE_F32); + psImage *model = psImageAlloc(image->numCols,image->numRows,PS_TYPE_F32); + + // Read header and construct original positions + psS32 naxis1 = psMetadataLookupS32(NULL, header, "NAXIS1"); + psS32 naxis2 = psMetadataLookupS32(NULL, header, "NAXIS2"); + psS32 imnaxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); + psS32 imnaxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); + psString ccdsum = psMetadataLookupStr(NULL, header, "CCDSUM"); + psS32 xbin = atoi(strtok(ccdsum," ")); + psS32 ybin = atoi(strtok(NULL, " ")); + psS32 xoffset = (naxis1 * xbin - imnaxis1) / (2 * xbin); + psS32 yoffset = (naxis2 * ybin - imnaxis2) / (2 * ybin); + + psPlane *pix = psPlaneAlloc(); // Pixel coordinates on chip + psPlane *fp = psPlaneAlloc(); // Focal plane coordinates + psPlane *tp = psPlaneAlloc(); // Tangent plane coordinates + psSphere *sky = psSphereAlloc(); // Sky coordinates + + for (v = 0; v < image->numRows; v++) { + pix->y = (v - yoffset) * ybin; + for (u = 0; u < image->numCols; u++) { + pix->x = (u - xoffset) * xbin; + psPlaneTransformApply(fp, chip->toFPA, pix); + psPlaneTransformApply(tp, smfFile->fpa->toTPA, fp); + psDeproject(sky, tp, smfFile->fpa->toSky); + + psProject(tp,sky,stack->fpa->toSky); + + raim->data.F32[v][u] = tp->x; + decim->data.F32[v][u] = tp->y; + model->data.F32[v][u] = 0.0; + + // Check the bounds so we'll know how large of an area to model in the map + if (tp->x < data->ra_min) { data->ra_min = tp->x; } + else if (tp->x > data->ra_max) { data->ra_max = tp->x; } + if (tp->y < data->dec_min) { data->dec_min = tp->y; } + else if (tp->y > data->dec_max) { data->dec_max = tp->y; } + + if (sky->r < rr) { rr = sky->r; } + else if (sky->r > RR) { RR = sky->r; } + if (sky->r < dd) { dd = sky->d; } + else if (sky->d > DD) { DD = sky->d; } + + } + } + + // Allocate the model data for this chip + psMetadata *this_model = psMetadataAlloc(); + + // Attach vectors to teh structure of the chip + psMetadataAddImage(this_model,PS_LIST_TAIL, + "bkg image", 0, + "ota space X vector", image); + psMetadataAddImage(this_model,PS_LIST_TAIL, + "bkg ra", 0, + "ota space ra vector", raim); + psMetadataAddImage(this_model,PS_LIST_TAIL, + "bkg dec", 0, + "ota space dec vector", decim); + psMetadataAddImage(this_model,PS_LIST_TAIL, + "bkg calibrated data", 0, + "ota space corrected data", model); + + // Define default background model parameters, using the assumption: + // observed = camera + offset + scale * astrophysical + psMetadataAddF32(this_model,PS_LIST_TAIL, + "bkg offset", PS_META_REPLACE, + "background offset for this exposure/ota pair", 0.0); + psMetadataAddF32(this_model,PS_LIST_TAIL, + "bkg scale", PS_META_REPLACE, + "background scale parameter for this exposure/ota pair", 1.0); + // Add this model to the current model + psMetadataAddMetadata(Bmodel,PS_LIST_TAIL, + chipName, PS_META_REPLACE, + "model data for this exposure/ota", this_model); + // Free model data for this chip + psFree(modelFileName); + psFree(modelFits); + psFree(header); + psFree(pix); + psFree(fp); + psFree(tp); + psFree(sky); + psFree(image); + psFree(raim); + psFree(decim); + psFree(model); + psFree(this_model); + + // Check to see if we've loaded or allocated an OTA solution container for this chip + if (!psMetadataLookupPtr(NULL,data->OTA_solutions,chipName)) { // No solution metadata entry exists for this chipName + // FIX this should try to find the imagefile. + if (data->fit_OTAS) { // We are fitting OTAs, so allocate a new image + psImage *solution = psImageAlloc(image->numCols,image->numRows,PS_TYPE_F32); + psMetadataAddPtr(data->OTA_solutions,PS_LIST_TAIL, + chipName, PS_DATA_UNKNOWN | PS_META_REPLACE, + "OTA solution element", solution); + } + else { // We are not fitting OTAs, so read the one that should be saved on OTApath. + psString solutionFileName = psStringCopy(data->OTApath); + psStringAppend(&solutionFileName, ".%s.fits",chipName); + psFits *solutionFits = psFitsOpen(solutionFileName,"r"); + psImage *solution = psFitsReadImage(solutionFits,psRegionSet(0,0,0,0),0); + psMetadataAddImage(data->OTA_solutions,PS_LIST_TAIL, + chipName, 0, + "OTA solution element", solution); + psFree(solutionFits); + psFree(solutionFileName); + psFree(solution); + } + } + + + psMemStats(0,&A,&P); + fprintf(stderr,"chip2 %ld %ld\n",A,P); + +/* if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { */ +/* psError(psErrorCodeLast(), false, "Error saving data to files."); */ +/* return false; */ +/* } */ + } // end chip loop + + // Add the set of models to the datastructure + psMetadataAddMetadata(data->models, PS_LIST_TAIL, + smfFile->origname, PS_META_REPLACE, + "model data for this exposure", Bmodel); + psMetadataAddS16(data->models,PS_LIST_TAIL, + "N", PS_META_REPLACE, + "counter", i); + // data->models->name->XY__->{image/ra/dec/calibrated/offset/scale} + // data->models->counter->XY__->{image/ra/dec/calibrated/offset/scale} + + + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(psErrorCodeLast(), false, "Error saving data to files."); + return false; + } + pmFPAfileActivate(config->files, false, NULL); + psFree(modelContent); + // psFree(smfFile); + psFree(smfFileName); + psFree(view); + + psFree(config->files); + config->files = psMetadataAlloc(); + // + + // + } // end smf loop + + // Unlie now. + psFree(config->files); + config->files = config_files; + + // Allocate the modelMap for the region we're covering. + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psImageBinning *binning = psImageBinningAlloc(); + binning->nXruff = 33; // Number of samples + binning->nYruff = 33; + binning->nXfine = ceil(data->ra_max - data->ra_min) + 1; // This is the range we're looking at + binning->nYfine = ceil(data->dec_max - data->dec_min) + 1; + binning->nXskip = floor(data->ra_min); + binning->nYskip = floor(data->dec_min); + psImageBinningSetScale(binning,PS_IMAGE_BINNING_CENTER); + printf("Sizes: sky: %f %f -> %f %f :: tp: %f %f -> %f %f :: map: %d %d\n", + rr,dd,RR,DD,data->ra_min,data->dec_min,data->ra_max,data->dec_max,binning->nXfine,binning->nYfine); + data->modelMap = psImageMapNoImageAlloc( binning,stats); + + return true; +} Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackData.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackData.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackData.c (revision 36622) @@ -0,0 +1,78 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppBackgroundStack.h" + +// Destructor +static void backgroundDataFree(ppBackgroundStackData *data // Data to free + ) +{ + // CZW 2014-03-24 + // The things that are commented out here, and therefore don't get freed are in that state because + // I'd far rather spend time sorting out actual math issues than dealing with data structures. + + // psFree(data->contents); + +// psFree(data->smfs); + + psFree(data->models); + psFree(data->OTA_solutions); + psFree(data->OTApath); + + psFree(data->modelMap); + + + psFree(data->stack_data); + psFree(data->stacks); + psFree(data->outRoot); + //psFree(data->config); + return; +} + + +ppBackgroundStackData *ppBackgroundStackDataAlloc(void) +{ + ppBackgroundStackData *data = psAlloc(sizeof(ppBackgroundStackData)); // Processing data, to return + psMemSetDeallocator(data, (psFreeFunc)backgroundDataFree); + + data->contents = NULL; + data->smfs = psArrayAlloc(0); + + data->models = psMetadataAlloc(); + data->OTA_solutions = psMetadataAlloc(); + data->fit_OTAS = false; + data->OTApath = NULL; + + data->modelMap = NULL; + data->ra_min = 1e9; + data->ra_max = -1e9; + data->dec_min = 1e9; + data->dec_max = -1e9; + + data->stack_data = psArrayAlloc(0); + data->stacks = psArrayAlloc(0); + data->outRoot = NULL; + data->config = NULL; + + return data; +} + + +ppBackgroundStackData *ppBackgroundStackDataInit(int *argc, char **argv) +{ + PS_ASSERT_PTR_NON_NULL(argc, NULL); + PS_ASSERT_PTR_NON_NULL(argv, NULL); + + ppBackgroundStackData *data = ppBackgroundStackDataAlloc(); // Processing data, to return + data->config = pmConfigRead(argc, argv, PPBACKGROUND_RECIPE); + if (!data->config) { + psError(psErrorCodeLast(), false, "Unable to read configuration."); + return NULL; + } + return data; +} Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackLoop.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackLoop.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackLoop.c (revision 36622) @@ -0,0 +1,265 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppBackgroundStack.h" + +bool ppBackgroundStackLoop(ppBackgroundStackData *data // Run-time data + ) +{ + pmConfig *config = data->config; // Configuration data + int i; + // + // Solve the data into a consistent model. + { // This block does the initialization + // If we didn't load the OTA solution from an external source, we need to build one. + if (data->fit_OTAS) { + printf("Fitting OTAs!\n"); + if (!ppBackgroundStackModelFitOTASolution(data)) { + // Currently can't fail. + psError(psErrorCodeLast(), false, "Error calculating the OTA solution"); + return(false); + } + + if (data->OTApath) { + // This will write the solutions out. + psMetadataIterator *iter = psMetadataIteratorAlloc(data->OTA_solutions, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + int i = 0; + while ((item = psMetadataGetAndIncrement(iter))) { + i++; + if (item->type != PS_DATA_IMAGE) { + psString chipName = item->name; + psImage *image = item->data.V; + psMetadata *header = psMetadataAlloc(); + psString solutionFileName = psStringCopy(data->OTApath); + psStringAppend(&solutionFileName, ".%s.fits",chipName); + psString resolved = pmConfigConvertFilename(solutionFileName,config,true,true); + psFits *solutionFits = psFitsOpen(resolved,"w"); + if (!solutionFits) { + psError(2, false, "Unable to open FITS file %s to write model %s.", resolved, chipName); + psFitsClose(solutionFits); + psFree(resolved); + return(false); + } + if (!psFitsWriteImage(solutionFits, header, image, 0, NULL)) { + psError(2, false, "Unable to write FITS image %s.", resolved); + psFitsClose(solutionFits); + psFree(resolved); + return false; + } + if (!psFitsClose(solutionFits)) { + psError(2, false, "Unable to close FITS image %s.", resolved); + psFree(resolved); + return false; + } + psFree(resolved); + psFree(solutionFileName); + } + } + psFree(item); + psFree(iter); + } + + } + + + // This seems wrong, but I need a blank modelMap object, so we fit the zero-data we've stored in the calib objects + printf("Determining blank modelMap!\n"); + if (!ppBackgroundStackModelFit(data)) { + psError(psErrorCodeLast(), false, "Error determining the blank modelMap object."); + return(false); + } + + // Apply OTA solution + printf("Calib apply!\n"); + if (!ppBackgroundStackCalibApply(data)) { + psError(psErrorCodeLast(), false, "Error applying the calibration models."); + return(false); + } + } // End initialization block. + + // This is where a loop would likely start. + { + // Construct the offset information + printf("Model fit!\n"); + if (!ppBackgroundStackDataModelFit(data)) { + psError(psErrorCodeLast(), false, "Error determining the exposure/OTA scaling."); + return(false); + } + + // Apply full correction + printf("Calib apply!\n"); + if (!ppBackgroundStackCalibApply(data)) { + psError(psErrorCodeLast(), false, "Error applying the calibration models."); + return(false); + } + + // Fit the new model + printf("Determining model!\n"); + if (!ppBackgroundStackModelFit(data)) { + psError(psErrorCodeLast(), false, "Error determining the modelMap object."); + return(false); + } + } // End loop + + // Loop over the input images, and apply the models to construct the restored versions. + + for (i = 0; i < data->stack_data->n; i++) { + pmFPAfile *stack = data->stack_data->data[i]; + pmFPAview *view = pmFPAviewAlloc(0); + + // Define output image. Why is this always so hard to do? + pmFPA *tmp_fpa1,*tmp_fpa2; + tmp_fpa1 = pmFPAConstruct(config->camera,config->cameraName); + tmp_fpa2 = pmFPAConstruct(config->camera,config->cameraName); + + pmFPAfile *stack_model = pmFPAfileDefineOutput(config,tmp_fpa1,"PPBACKGROUND.STACK.MODEL"); + + if (!stack_model) { + psError(psErrorCodeLast(), false, "Unable to generate output model"); + return (false); + } + + pmFPAfile *stack_corr = pmFPAfileDefineOutput(config,tmp_fpa2,"PPBACKGROUND.STACK.OUTPUT"); + if (!stack_corr) { + psError(psErrorCodeLast(), false, "Unable to generate output result"); + return (false); + } + stack_model->save = true; + stack_corr->save = true; + + printf("I'm about to loop over the parts of this stack: %d\n",i); + // Iterate over the images. + pmFPAfileActivate(config->files,true,"PPBACKGROUND.STACK"); + pmFPAfileActivate(config->files,true,"PPBACKGROUND.STACK.MODEL"); + pmFPAfileActivate(config->files,true,"PPBACKGROUND.STACK.OUTPUT"); + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(psErrorCodeLast(), false, "load failure for Chip"); + return(false); + } + + + pmChip *chip; + while ((chip = pmFPAviewNextChip(view, stack->fpa, 1))) { + if (!chip->process || !chip->file_exists) { + continue; + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(psErrorCodeLast(), false, "load failure for Chip"); + return(false); + } + printf(" I'm in a chip\n"); + pmCell *cell; + + while ((cell = pmFPAviewNextCell(view, stack->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(psErrorCodeLast(), false, "load failure for Cell"); + return(false); + } + if (cell->readouts->n > 1) { + psWarning ("Skipping Video Cell for ppImageDetrendReadout"); + continue; + } + printf(" I'm in a cell\n"); + + + // process each of the readouts + pmReadout *readout; // Readout from cell + while ((readout = pmFPAviewNextReadout (view, stack->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(psErrorCodeLast(), false, "load failure for Readout"); + return(false); + } + if (!readout->data_exists) { + continue; + } + printf(" I'm in a readout\n"); + + // Futz with things to get an acceptable output product. + pmCell *model_cell = pmFPAviewThisCell(view,stack_model->fpa); + pmCell *corr_cell = pmFPAviewThisCell(view,stack_corr->fpa); + + pmReadout *model = pmReadoutAlloc(model_cell); + pmReadout *corr = pmReadoutAlloc(corr_cell); + model->data_exists = true; + corr->data_exists = true; + model->parent->data_exists = true; + corr->parent->data_exists = true; + model->parent->parent->data_exists = true; + corr->parent->parent->data_exists = true; + model->image = psImageAlloc(readout->image->numCols,readout->image->numRows,PS_TYPE_F32); + corr->image = psImageAlloc(readout->image->numCols,readout->image->numRows,PS_TYPE_F32); + + model_cell->concepts = psMemIncrRefCounter(cell->concepts); + model_cell->conceptsRead = cell->conceptsRead; + corr_cell->concepts = psMemIncrRefCounter(cell->concepts); + corr_cell->conceptsRead = cell->conceptsRead; + + psPlane *pix = psPlaneAlloc(); // Pixel coordinates on chip + psPlane *fp = psPlaneAlloc(); // Focal plane coordinates + psPlane *tp = psPlaneAlloc(); // Tangent plane coordinates + + int x,y; + for (y = 0; y < readout->image->numRows; y++) { + pix->y = y; + for (x = 0; x < readout->image->numCols; x++) { + pix->x = x; + // Calculate model for each pixel of output + // psPlaneTransformApply(fp, chip->toFPA, pix); + psPlaneTransformApply(tp, stack->fpa->toTPA, pix); + + model->image->data.F32[y][x] = psImageMapEval(data->modelMap,tp->x,tp->y); + corr->image->data.F32[y][x] = readout->image->data.F32[y][x] + model->image->data.F32[y][x]; + + } + } + psFree(pix); + psFree(fp); + psFree(tp); + } // Close readout + printf(" I'm done with that readout\n"); + // Close output image + } // Close Cell + // Close cells (XXX shouldn't pmFPAfileClose iterate down as needed?) + view->cell = -1; + while ((cell = pmFPAviewNextCell(view, stack->fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(psErrorCodeLast(), false, "save failure for Cell"); + return(false); + } + } + + // Close chip + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(psErrorCodeLast(), false, "save failure for Chip"); + return(false); + } + } // Close chip. + // Output and Close FPA + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(psErrorCodeLast(), false, "save failure for FPA"); + return(false); + } + psFree(view); + } + + + return(true); +} + + + Index: /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackMath.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackMath.c (revision 36622) +++ /branches/eam_branches/ipp-20140226/ppBackground/src/ppBackgroundStackMath.c (revision 36622) @@ -0,0 +1,215 @@ +/** @file ppBackgroundStackArguments.c + * + * @brief + * + * @ingroup ppBackgroundStack + * + * @author Paul Price + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppBackgroundStack.h" + +// data->models->name->XY__->{image/ra/dec/calibrated/offset/scale} +// data->models->counter->XY__->{image/ra/dec/calibrated/offset/scale} + +// +// Determine the best OTA-to-OTA solution for this set of exposures/chips. +// This is done by finding the median value of the set of (ota,u,v) values for +// all exposures. +// solution_{OTA}(u,v) = < data_{OTA}(u,v) >_{exposures} +bool ppBackgroundStackModelFitOTASolution(ppBackgroundStackData *data) { + int u,v; + psMetadataIterator *iter = psMetadataIteratorAlloc(data->OTA_solutions, PS_LIST_HEAD, NULL); // Iterate over all chips. + psMetadataItem *item; + psStats *stats = psStatsAlloc( PS_STAT_ROBUST_MEDIAN ); + psS16 N = psMetadataLookupS16(NULL, data->models, "N"); + + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_UNKNOWN) { + psWarning("This seems like I have a not-data, when I expect a data."); + continue; + } + psString workingChip = item->name; + psImage *solution = item->data.V; + + for (v = 0; v < solution->numRows; v++) { + for (u = 0; u < solution->numCols; u++) { + + psVector *tmp = psVectorAllocEmpty(N,PS_TYPE_F32); + + psMetadataIterator *expIter = psMetadataIteratorAlloc(data->models, PS_LIST_HEAD, NULL); + psMetadataItem *expItem; + while ((expItem = psMetadataGetAndIncrement(expIter))) { + if (expItem->type != PS_DATA_METADATA) { + continue; // This is the N counter + } + psMetadata *chipData = psMetadataLookupPtr(NULL, expItem->data.md, workingChip); + psImage *image = psMetadataLookupPtr(NULL, chipData, "bkg image"); + psVectorAppend(tmp,image->data.F32[v][u]); + } // End loop over exposures + psStatsInit(stats); + psVectorStats(stats,tmp,NULL,NULL,0); + solution->data.F32[v][u] = stats->robustMedian; + + psFree(expIter); + } // End u + } // End v + } // End working chip scan + + psFree(iter); + psFree(stats); + return(true); +} + +// +// Determine the relative offset of this exposure/ota relative to the current estimate of the sky at this point. +// Note: the initial assumption of the model is a flat grid of zero, "the sky contains nothing real." +// SOLVE: model(r,d) = scale * data(r,d) + offset + solution_{OTA}(u,v) +// FOR: scale, offset +bool ppBackgroundStackDataModelFit(ppBackgroundStackData *data) { + int u,v; + + psMetadataIterator *expIter = psMetadataIteratorAlloc(data->models, PS_LIST_HEAD, NULL); + psMetadataItem *expItem; + while ((expItem = psMetadataGetAndIncrement(expIter))) { + if (expItem->type != PS_DATA_METADATA) { + continue; // This is the N counter + } + + psMetadataIterator *chipIter = psMetadataIteratorAlloc(expItem->data.md, PS_LIST_HEAD, NULL); + psMetadataItem *chipItem; + while ((chipItem = psMetadataGetAndIncrement(chipIter))) { + psImage *image = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg image"); + psImage *ra = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg ra"); + psImage *dec = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg dec"); + + psVector *obs = psVectorAlloc(image->numRows * image->numCols, PS_TYPE_F32); + psVector *model= psVectorAlloc(image->numRows * image->numCols, PS_TYPE_F32); + + for (v = 0; v < image->numRows; v++) { + for (u = 0; u < image->numCols; u++) { + obs->data.F32[v * image->numCols + u] = image->data.F32[v][u]; + model->data.F32[v * image->numCols + u] = psImageMapEval(data->modelMap,ra->data.F32[v][u],dec->data.F32[v][u]); + } + } + + psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD,1); + int status = psVectorFitPolynomial1D(poly,NULL,0,model,NULL,obs); + if (!status) { + psMetadataAddF32(chipItem->data.md,PS_LIST_TAIL,"bkg offset", PS_META_REPLACE, "background offset for this exposure/ota pair", poly->coeff[0]); + psMetadataAddF32(chipItem->data.md,PS_LIST_TAIL,"bkg scale", PS_META_REPLACE, "background scale for this exposure/ota pair", poly->coeff[1]); + } + psFree(poly); + } // End OTA loop + psFree(chipIter); + } // End smf/exp loop + psFree(expIter); + return(true); +} + + +// +// Apply the corrections to attempt to put all exposures, and all OTAs onto a common level, +// such that they should all match the true sky +// calib_{exposure,OTA}(r,d) = scale * data_{exposure,OTA}(r,d) + offset + solution_{OTA}(u,v) +bool ppBackgroundStackCalibApply(ppBackgroundStackData *data) { + int u,v; + + psMetadataIterator *expIter = psMetadataIteratorAlloc(data->models, PS_LIST_HEAD, NULL); + psMetadataItem *expItem; + while ((expItem = psMetadataGetAndIncrement(expIter))) { + if (expItem->type != PS_DATA_METADATA) { + continue; // This is the N counter + } + psMetadataIterator *chipIter = psMetadataIteratorAlloc(expItem->data.md, PS_LIST_HEAD, NULL); + psMetadataItem *chipItem; + while ((chipItem = psMetadataGetAndIncrement(chipIter))) { + const char *chipName = chipItem->name; + + psImage *image = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg image"); + psImage *model = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg calibrated data"); + psImage *camera= psMetadataLookupPtr(NULL, data->OTA_solutions, chipName); + + psF32 offset = psMetadataLookupF32(NULL,chipItem->data.md,"bkg offset"); + psF32 scale = psMetadataLookupF32(NULL,chipItem->data.md,"bkg scale"); + + for (v = 0; v < image->numRows; v++) { + for (u = 0; u < image->numCols; u++) { + model->data.F32[v][u] = scale * image->data.F32[v][u] - offset - camera->data.F32[v][u]; + } + } + } // End chip + psFree(chipIter); + } // End smf + psFree(expIter); + return(true); +} + +// +// Determine the best estimate of the true sky from the ensemble of calibrated samples: +// model(r,d) = < calib(r,d) >_{exposures,OTAs} +// This "averaging" is done using the psImageMapClipFit. +bool ppBackgroundStackModelFit(ppBackgroundStackData *data) { + long j; + int u,v; + + psS16 N = psMetadataLookupS16(NULL, data->models, "N"); + psVector *X = psVectorAlloc(N * 60 * 13 * 13,PS_TYPE_F32); + psVector *Y = psVectorAlloc(N * 60 * 13 * 13,PS_TYPE_F32); + psVector *Z = psVectorAlloc(N * 60 * 13 * 13,PS_TYPE_F32); + psVector *E = psVectorAlloc(N * 60 * 13 * 13,PS_TYPE_F32); + psVector *mask = psVectorAlloc(N * 60 * 13 * 13,PS_TYPE_VECTOR_MASK); + j = 0; + + pmFPAview *view = pmFPAviewAlloc(0); + psStats *stats = psStatsAlloc( PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV ); + + psMetadataIterator *expIter = psMetadataIteratorAlloc(data->models, PS_LIST_HEAD, NULL); + psMetadataItem *expItem; + while ((expItem = psMetadataGetAndIncrement(expIter))) { + if (expItem->type != PS_DATA_METADATA) { + continue; // This is the N counter + } + psMetadataIterator *chipIter = psMetadataIteratorAlloc(expItem->data.md, PS_LIST_HEAD, NULL); + psMetadataItem *chipItem; + while ((chipItem = psMetadataGetAndIncrement(chipIter))) { + psImage *calib = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg calibrated data"); + psImage *ra = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg ra"); + psImage *dec = psMetadataLookupPtr(NULL, chipItem->data.md, "bkg dec"); + for (v = 0; v < calib->numRows; v++) { + for (u = 0; u < calib->numCols; u++) { + X->data.F32[j] = ra->data.F32[v][u]; + Y->data.F32[j] = dec->data.F32[v][u]; + Z->data.F32[j] = calib->data.F32[v][u]; + E->data.F32[j] = 1.0; + mask->data.PS_TYPE_VECTOR_MASK_DATA[j] = 0; + j++; + } + } + } // End chip + psFree(chipIter); + } // End smfs + + bool fitStatus; + bool status = psImageMapClipFit(&fitStatus,data->modelMap,stats, mask, 0, X, Y, Z, E); + + psFree(expIter); + psFree(X); + psFree(Y); + psFree(Z); + psFree(E); + psFree(mask); + psFree(stats); + psFree(view); + return(status); +} Index: /branches/eam_branches/ipp-20140226/ppStack/src/ppStackReadout.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppStack/src/ppStackReadout.c (revision 36621) +++ /branches/eam_branches/ipp-20140226/ppStack/src/ppStackReadout.c (revision 36622) @@ -184,4 +184,6 @@ psFree(stack); + psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum); + sectionNum++; @@ -293,4 +295,6 @@ psFree(stack); + psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum); + sectionNum++; Index: /branches/eam_branches/ipp-20140226/ppSub/src/ppSubMatchPSFs.c =================================================================== --- /branches/eam_branches/ipp-20140226/ppSub/src/ppSubMatchPSFs.c (revision 36621) +++ /branches/eam_branches/ipp-20140226/ppSub/src/ppSubMatchPSFs.c (revision 36622) @@ -156,8 +156,13 @@ psLogMsg("ppSub", PS_LOG_INFO, "Input FWHM: %f\nReference FWHM: %f\n", inFWHM, refFWHM); if (!isfinite(inFWHM) || !isfinite(refFWHM)) { +#ifdef SET_QUALITY_INSTEAD_OF_ERROR psErrorStackPrint(stderr, "Cannot determine FHWM for images, giving up."); int error = psErrorCodeLast(); // Error code ppSubDataQuality(data, error, PPSUB_FILES_ALL); return true; +#else + psError(PPSUB_ERR_DATA, false, "Cannot determine FHWM for images, giving up."); + return false; +#endif } Index: /branches/eam_branches/ipp-20140226/pstamp/scripts/pstamp_insert_request.pl =================================================================== --- /branches/eam_branches/ipp-20140226/pstamp/scripts/pstamp_insert_request.pl (revision 36621) +++ /branches/eam_branches/ipp-20140226/pstamp/scripts/pstamp_insert_request.pl (revision 36622) @@ -45,4 +45,5 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); +my $fhead = can_run('fhead') or (warn "Can't find fhead" and $missing_tools = 1); if ($missing_tools) { @@ -56,22 +57,62 @@ # Note that if it's a pstamp request then REQ_NAME should be defined. # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID - my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID"; + # my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID"; + my $command = "$fhead -x 0 $tmp_req_file"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); -if (0) { - # stoopid fields doesn't set exit status to zero when it works unless ($success) { print STDERR @$stderr_buf; exit $error_code >> 8; } -} my $output = join "", @$stdout_buf; - (undef, $extname, $extver, $req_name) = split " ", $output; + + + my $makehash = 0; + my %hash; + foreach my $line (split "\n", $output) { + chomp $line; + # split lines inte left and right using equals sign. left is the keyword + my ($key, $right) = split "=", $line; + # skip if there was no '=' + next if !$right; + + $key =~ s/ //g; + + # separate value from comment + my ($value, $comment) = split "/", $right; + # remove ' and space characters from key and value + $value =~ s/'//g; + $value =~ s/ //g; + + # print "$key $value\n"; + + # extract the values that we are looking for + $extname = $value if ($key eq "EXTNAME"); + $extver = $value if ($key eq "EXTVER"); + $req_name = $value if ($key eq "REQ_NAME"); + $req_name = $value if ($key eq "QUERY_ID"); + + # optionally build hash. Duplicate keys get the last value seen + if ($makehash) { + $hash{$key} = $value; + } + } + + # (undef, $extname, $extver, $req_name) = split " ", $output; + if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) { print STDERR "invalid request file\n"; + print "invalid request file\n"; exit $PS_EXIT_DATA_ERROR; } + if (!defined $req_name) { print STDERR "invalid request file no REQ_NAME or QUERY_ID\n"; + print "invalid request file no REQ_NAME or QUERY_ID\n"; + exit $PS_EXIT_DATA_ERROR; + } + if (!defined $extver) { + print STDERR "invalid request file no EXTVER found\n"; + print "invalid request file no EXTVER found\n"; exit $PS_EXIT_DATA_ERROR; } @@ -119,4 +160,5 @@ exit 0; + # Ask the database for the next web request number sub get_webreq_num Index: /branches/eam_branches/ipp-20140226/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/eam_branches/ipp-20140226/pstamp/scripts/pstampparse.pl (revision 36621) +++ /branches/eam_branches/ipp-20140226/pstamp/scripts/pstampparse.pl (revision 36622) @@ -243,5 +243,7 @@ my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/)); -my $big_limit = 100; # XXX: this should be in a configuration file somewhere not hard coded +# XXX: these should be in a configuration file somewhere not hard coded +my $big_limit = 100; +my $job_big_limit = 400; if ($watch_for_big_requests and $nRows > $big_limit) { @@ -267,5 +269,5 @@ # see whether number of jobs limit for high priority request was exceeded - if ($watch_for_big_requests and $num_jobs > $big_limit) { + if ($watch_for_big_requests and $num_jobs > $job_big_limit) { $label = change_to_lower_priority_label($label); $watch_for_big_requests = 0; @@ -311,4 +313,5 @@ my $rownum = $row->{ROWNUM}; if (!validID($rownum)) { + $rownum = 'NULL' if !defined $rownum; print STDERR "$rownum is not a valid ROWNUM\n"; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); @@ -316,5 +319,6 @@ } my $job_type = $row->{JOB_TYPE}; - if (($job_type ne "stamp") and ($job_type ne "get_image")) { + if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) { + $job_type = 'NULL' if !defined $job_type; print STDERR "$job_type is not a valid JOB_TYPE\n"; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST); @@ -325,4 +329,5 @@ if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and ($req_type ne "byskycell") and ($req_type ne "bydiff")) { + $req_type = 'NULL' if !defined $req_type; print STDERR "$req_type is not a valid REQ_TYPE\n"; insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);