Index: trunk/ippScripts/scripts/dist_component.pl =================================================================== --- trunk/ippScripts/scripts/dist_component.pl (revision 26030) +++ trunk/ippScripts/scripts/dist_component.pl (revision 26032) @@ -53,5 +53,5 @@ my $missing_tools; my $disttool = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1); -my $dist_make_bundle = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1); +my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -103,19 +103,152 @@ } - -my ($file_name, $bytes, $md5sum); -{ - my $command = "$dist_make_bundle --dist_id $dist_id --camera $camera --stage $stage --stage_id $stage_id"; - $command .= " --component $component --path_base $path_base --outdir $outdir"; - $command .= " --state $run_state" if defined $run_state; - $command .= " --data_state $data_state" if defined $data_state; - $command .= " --poor_quality" if defined $poor_quality; - $command .= " --no_magic" if defined $no_magic; - $command .= " --magicked" if defined $magicked; - $command .= " --clean" if defined $clean; - $command .= " --save-temps" if defined $save_temps; - $command .= " --dbname $dbname" if defined $dbname; - $command .= " --verbose" if defined $verbose; - +$ipprc->define_camera($camera); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# making a clean bundle of raw images doesn't make sense + +# create the output directories if it is not a nebulous path and it doesn't exist +if (index($outdir, "neb://") != 0) { + if (! -e $outdir ) { + my $code = system "mkdir -p $outdir"; + &my_die("cannot create output directory $outdir", $dist_id, $component, + $code >> 8) if $code; + } +} + +# Get the list of data products for this component +# note: We my_die in get_file_list if something goes wrong. + +my $file_list = get_file_list($stage, $component, $path_base, $clean); + +if (($stage ne 'raw') and ($stage ne 'fake')) { + # If the file list is empty it is an error because we should at least get a config dump file + # except for fake stage which doesn't do anything yet + &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list); +} + +# set up directory for temporary files + +my $tmpdir = "$outdir/tmpdir.$dist_id.$component"; +if (-e $tmpdir) { + if (-d $tmpdir) { + my $rc = system "rm -r $tmpdir"; + &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc; + } else { + unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR); + } +} +mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component, + $PS_EXIT_UNKNOWN_ERROR); + +# +# we need to run set masked pixels to NAN in the image and variance images +# unless +# 1. we are building a clean distribution bundle +# 2. magic is not required for this distRun +# 3. the processing for the component produced no images (warp or diff with bad quality for example) +my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality); + +my ($image, $mask, $variance); +my ($inv_image, $inv_mask, $inv_variance); + +# foreach my $file_rule (keys %$file_list) { +my $num_files = 0; +foreach my $file (@$file_list) { + # check whether this file rule refers to an image, mask, or variance fits image + my $file_rule = $file->{file_rule}; + my $image_type = get_image_type($stage, $file_rule); + + # if this is an image and we are building a clean bundle or if quality is bad skip this rule + next if $image_type and ($clean or $poor_quality); + + # if magic is required, don't ship jpegs or binned fits images + next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/) + or ($file_rule =~ /.JPEG2/)); + + if ($stage eq "diff") { + next if $file_rule =~ /CONV/; + } + + my $file_name = $file->{name}; + my $base = basename($file_name); + my $path = $ipprc->file_resolve($file_name); + + if (!$path) { + # skip this file if $poor_quality + # this is for compatability with older runs which don't have the files list in the + # config dump. + # Once we give up on supporting that we can remove the next line. (If the file is in the list + # it must exist) + next if $poor_quality; + + &my_die("failed to resolve $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR); + } + + # open the file to make sure it exists (and to work around the failed mount phenomena) + my $fh = open_with_retries($path); + close $fh; + + # we need to pre-process the image before adding to the bundle. Save the path names. + # the images will be created below + $num_files++; + if ($image_type && $nan_masked_pixels) { + # save the + if ($image_type eq 'image') { + $image = $file_name; + } elsif ($image_type eq 'mask') { + $mask = $file_name; + } elsif ($image_type eq 'variance') { + $variance = $file_name;; + } elsif ($image_type eq 'inv_image') { + $inv_image = $file_name; + } elsif ($image_type eq 'inv_mask') { + $inv_mask = $file_name;; + } elsif ($image_type eq 'inv_variance') { + $inv_variance = $file_name;; + } else { + &my_die("invalid image type found: $image_type", $dist_id, $component, + $PS_EXIT_PROG_ERROR); + } + } else { + # create a symbolic link from the file in the nebulous repository + # in the temporary directory + symlink $path, "$tmpdir/$base"; + } +} + +if ($nan_masked_pixels) { + # One last check as to whether magic has been applied to the inputs + + # Note: the sql for disttool -pendingcomponent won't select a component that + # requires magic and hasn't been magicked, but we check again here + + if (!($magicked or $no_magic)) { + &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR); + } + + &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image; + &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask; + &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance; + + my $class_id; + # run streaksrelease to set masked pixels to NAN + if ($stage eq "raw") { + $class_id = $component; + # we can use the chip mask because disttool demands that magic have been run + # and so the camera mask and the chip mask are the same + $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component); + my $fh = open_with_retries($mask); + close $fh; + } elsif ($stage eq "chip") { + $class_id = $component; + } + + my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir"; + $command .= " -class_id $class_id" if $class_id; + $command .= " -mask $mask" if $mask; + $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask); + $command .= " -weight $variance" if $variance; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -124,9 +257,53 @@ &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); } - # parse stdout_buf to get ($file_name, $bytes, $md5sum); -} - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - + if ($inv_image) { + $command = "$streaksrelease -stage $stage -image $inv_image -outroot $tmpdir"; + $command .= " -mask $inv_mask" if $inv_mask; + $command .= " -weight $inv_variance" if $inv_variance; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); + } + } +} + + +my $file_name; +my $bytes; +my $md5sum; + +if ($num_files) { + # create the tarfile + # XXX TODO: create a file rule for the tar file name + my $tbase = basename($path_base); + $tbase .= ".$component" if $component; + $file_name = "$tbase.tgz"; + my $tarfile = "$outdir/$file_name"; + + my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile ."; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); + } + + # tell the module not to die on error + $Digest::MD5::File::NOFATALS = 1; + $md5sum = file_md5_hex($tarfile); + &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum; + + $bytes = -s $tarfile; + + delete_tmpdir($tmpdir); +} else { + # no files for this component + $file_name = "none"; + $bytes = 0; + $md5sum = "0"; +} { my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir"; @@ -149,4 +326,159 @@ ### Pau. + +# return the image type (image, mask, or variance) if this file rule refers to +# one of the big fits files that are not included in a clean distribution +sub get_image_type { + my $stage = shift; + my $rule = shift; + my $type; + + if ($stage eq "raw") { + if ($rule eq "RAW.IMAGE") { + $type = 'image'; + } + } elsif ($stage eq "chip") { + $type = $chip_cleaned{$rule}; + } elsif ($stage eq "camera") { + $type = $camera_cleaned{$rule}; + } elsif ($stage eq "fake") { + $type = $fake_cleaned{$rule}; + } elsif ($stage eq "warp") { + $type = $warp_cleaned{$rule}; + } elsif ($stage eq "diff") { + $type = $diff_cleaned{$rule}; + } elsif ($stage eq "stack") { + $type = $stack_cleaned{$rule}; + } else { + &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); + } + return $type; +} + +sub open_with_retries { + my $name = shift; + + my $tries = 1; + my $max_tries = 5; + my $opened = 0; + while (!$opened && ($tries <= $max_tries)) { + $opened = open(IN, "<$name"); + if (!$opened) { + print STDERR "WARNING failed to open $name re-try $tries\n"; + $tries++; + sleep 5; + } + } + + &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component, + $PS_EXIT_DATA_ERROR) if (!$opened); + + return *IN; +} + +sub get_file_list { + my $stage = shift; + my $component = shift; + my $path_base = shift; + my $clean = shift; + + my @file_list; + if ($stage eq "raw") { + # XXX: TODO for now disttool sets path_base is set to the uri of the file + # eventually rawImfile will have a path_base that we'll need to add '.fits' + # XXX do we want to distribute the registration log files? + if (!$clean) { + my %file; + $file{file_rule} = "RAW.IMAGE"; + $file{name} = $path_base; + push @file_list, \%file; + } + return \@file_list; + } + + # we get the list of output data products for this run from the config dump file that + # is created when the run is done + my $config_file_rule; + if ($stage eq "chip") { + $config_file_rule = "PPIMAGE.CONFIG"; + } elsif ($stage eq "camera") { + $config_file_rule = "PSASTRO.CONFIG"; + } elsif ($stage eq 'fake') { + # XXX: fake is a no op now return an emtpy list + return \@file_list; + } elsif ($stage eq "warp") { + $config_file_rule = "PSWARP.CONFIG"; + } elsif ($stage eq "diff") { + $config_file_rule = "PPSUB.CONFIG"; + } elsif ($stage eq "stack") { + $config_file_rule = "PPSTACK.CONFIG"; + } else { + &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); + } + my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or + &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR); + + my %config_file_hash; + + # add the configuration file to the list + $config_file_hash{file_rule} = $config_file_rule; + $config_file_hash{name} = $config_file; + push @file_list, \%config_file_hash; + + my $resolved = $ipprc->file_resolve($config_file); + + &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR) if (!$resolved); + + # we don't use the mdc parser because the perl parser is way is too slow for complicated config + # files like this + my $in = open_with_retries($resolved); + + my $line; + while ($line = <$in>) { + if ($line =~ /FILES.OUTPUT/) { + # print "found FILES.OUTPUT\n"; + last; + } + } + &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR) if (!$line); + + while ($line = <$in>) { + chomp $line; + my ($key, $type, $val) = split " ", $line; + # skip blank lines + next if !$key; + # we're done when we find END + last if $key eq "END"; + # skip multi and other lines + next if ($type ne "STR"); + + # printf "%-32.32s %s\n", $key, $val; + + &my_die("no value found for file rule $key in $resolved", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR) if (!$val); + + my %file; + $file{file_rule} = $key; + $file{name} = $val; + push @file_list, \%file; + } + close $in; + + return \@file_list; +} + + + + +sub file_check +{ + my $file = shift; # Name of file + &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists($file); +} + sub my_die { @@ -175,7 +507,16 @@ } + delete_tmpdir(); + carp($msg); exit $exit_code; } +sub delete_tmpdir +{ + if (!$save_temps and $tmpdir and -e $tmpdir) { + system "rm -r $tmpdir"; + } +} + __END__