Index: trunk/ippScripts/scripts/dist_component.pl =================================================================== --- trunk/ippScripts/scripts/dist_component.pl (revision 23688) +++ trunk/ippScripts/scripts/dist_component.pl (revision 23693) @@ -83,9 +83,7 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my $basename = basename($path_base); - # making a clean bundle of raw images doesn't make sense if (($stage eq "raw") and $clean) { - # well I suppose we could ship the log file.... + # well I suppose we could ship the registration log file. naw. &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); } @@ -100,10 +98,10 @@ } -my $file_list = get_file_list($clean, $stage, $path_base); - -&my_die("failed to compute product list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$file_list; - -&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !scalar @$file_list; - +my $file_list = get_file_list($stage, $component, $path_base); + +# we die in get_file_list if something goes wrong. +# If the file list is empty it is an error because we should at least get a config dump file + +&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar keys %$file_list); # set up directory for temporary files @@ -121,21 +119,66 @@ $PS_EXIT_UNKNOWN_ERROR); +# XXX get this from camera and a recipe file +# or this with the quality flag +my $need_to_nan_masked = !$clean && ($camera eq "GPC1"); + +my ($image, $mask, $variance); + my @base_list; -foreach my $file (@$file_list) { - my $base = basename($file); - my $path = $ipprc->file_resolve($file); - # if file is not found, just skip it. - # This is not always the right thing to do. - # For example if warpSkyfile is ignored, there won't be many of the data products - # the new config dump will only list the files actually produced. - next if !$path; - +foreach my $file_rule (keys %$file_list) { + # if this is an image, mask, or variance image and we are building a clean distribution bundle, + # skip it + my $image_type = get_image_type($stage, $file_rule); + next if $clean && $image_type; + + my $file_name = $file_list->{$file_rule}; + my $base = basename($file_name); + my $path = $ipprc->file_resolve($file_name); + + &my_die("failed to resolve $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR) if !$path; + + # open the file to make sure it exists (and to work around the failed mount phenomena) + my $fh = open_with_retries($file_name); + close $fh; + + if ($image_type && $need_to_nan_masked) { + # save the + if ($image_type eq 'image') { + $image = $path + } elsif ($image_type eq 'mask') { + $mask = $path; + } elsif ($image_type eq 'variance') { + $variance = $path; + } else { + &my_die("invalid image type found: $image_type", $dist_id, $component, + $PS_EXIT_PROG_ERROR); + } + } else { + # just symlink the existing file into the temporary directory + symlink $path, "$tmpdir/$base"; + } push @base_list, $base; - symlink $path, "$tmpdir/$base"; -} - -if (!$clean) { - +} + +my $run_streaksrelease; +if ($need_to_nan_masked) { + # make sure that the needed images exist. + # XXX: they won't be found for a warp or diff with bad quality + # we probably should be passing in the component's quality flag + if (0) { + if (($stage ne "warp") and ($stage ne "diff")) { + &my_die("no image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image; + &my_die("no mask image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask; + &my_die("no variance image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance; + } + } + $run_streaksrelease = 1 if $image; +} + +if ($run_streaksrelease) { # Here is where we determine whether or not a GPC1 image can be released. + # XXX: actually the sql for disttool -pendingcomponent won't pass a component that + # requires magic and hasn't been magicked, but we check again here + # XXX: get this from a recipe if ($camera eq "GPC1" and !$magicked) { @@ -143,28 +186,16 @@ } + my $class_id; # run streaksrelease to set masked pixels to NAN - my ($image, $mask, $variance, $class_id); if ($stage eq "raw") { $class_id = $component; - $image = $path_base; - $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $class_id); + # we can use the chip mask because disttool demands that magic have been run + # and so the camera mask and the chip mask are the same + $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component); + my $fh = open_with_retries($mask); + close $fh; } elsif ($stage eq "chip") { $class_id = $component; - $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); - $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); - $variance = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id); - } elsif ($stage eq "warp") { - $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base); - $variance = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base); - } elsif ($stage eq "diff") { - $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base); - $variance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base); - } else { - &my_die("not ready for stage: $stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); - } - - push @base_list, $image; - push @base_list, $mask if $mask; - push @base_list, $variance if $variance; + } my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir"; @@ -234,20 +265,151 @@ ### Pau. +# filerules that are not included in 'cleaned' distribution bundles +# I could simplify the function below by using a two level hash but +# that would be less clear I think +my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image', + 'PPIMAGE.CHIP.MASK' => 'mask', + 'PPIMAGE.CHIP.VARIANCE' => 'variance' ); +my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' ); +my %fake_cleaned; +my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image', + 'PSWARP.OUTPUT.MASK' => 'mask', + 'PSWARP.OUTPUT.VARIANCE' => 'variance' ); +my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image', + 'PPSUB.OUTPUT.MASK' => 'mask', + 'PPSUB.OUTPUT.VARIANCE' => 'variance' ); +my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image', + 'PPSTACK.OUTPUT.MASK' => 'mask', + 'PPSTACK.OUTPUT.VARIANCE' => 'variance' ); + +# return the image type (image, mask, or variance) if this file rule refers to +# one of the big fits files that are not included in a clean distribution +sub get_image_type { + my $stage = shift; + my $rule = shift; + my $type; + + if ($stage eq "raw") { + if ($rule eq "RAW.IMAGE") { + $type = 'image'; + } + } elsif ($stage eq "chip") { + $type = $chip_cleaned{$rule}; + } elsif ($stage eq "camera") { + $type = $camera_cleaned{$rule}; + } elsif ($stage eq "fake") { + $type = $fake_cleaned{$rule}; + } elsif ($stage eq "warp") { + $type = $warp_cleaned{$rule}; + } elsif ($stage eq "diff") { + $type = $diff_cleaned{$rule}; + } elsif ($stage eq "stack") { + $type = $stack_cleaned{$rule}; + } else { + &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); + } + return $type; +} + +sub open_with_retries { + my $name = shift; + + my $tries = 1; + my $max_tries = 5; + my $opened = 0; + while (!$opened && ($tries <= $max_tries)) { + $opened = open(IN, "<$name"); + if (!$opened) { + print STDERR "WARNING failed to open $name re-try $tries\n"; + $tries++; + sleep 5; + } + } + + &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component, + $PS_EXIT_DATA_ERROR) if (!$opened); + + return *IN; +} sub get_file_list { - my $clean = shift; my $stage = shift; + my $component = shift; my $path_base = shift; - my @file_list; + my %file_list; if ($stage eq "raw") { - push @file_list, $path_base; + # XXX: TODO for now path_base is set to the uri of the file + # eventually we'll have a path_base that we'll need to resolve + $file_list{"RAW.IMAGE"} = $path_base; + return \%file_list; + } + + # we get the list of output data products for this run from the config dump file that + # is created when the run is done + my $config_file_rule; + if ($stage eq "chip") { + $config_file_rule = "PPIMAGE.CONFIG"; + } elsif ($stage eq "camera") { + $config_file_rule = "PSASTRO.CONFIG"; + } elsif ($stage eq "warp") { + $config_file_rule = "PSWARP.CONFIG"; + } elsif ($stage eq "diff") { + $config_file_rule = "PPSUB.CONFIG"; + } elsif ($stage eq "stack") { + $config_file_rule = "PPSTACK.CONFIG"; } else { - # TODO: these data will eventually come from the CONFIG dump - + &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); + } + my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or + &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR); + + my $resolved = $ipprc->file_resolve($config_file); + + &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR) if (!$resolved); + + # we don't use the mdc parser because the perl parser is way is too slow for complicated config + # files like this + my $in = open_with_retries($resolved); + + my $line; + while ($line = <$in>) { + if ($line =~ /FILES.OUTPUT/) { + # print "found FILES.OUTPUT\n"; + last; + } + } + + if ($line) { + # this is a new style config dump that contains the list of files + + my %files; + while ($line = <$in>) { + chomp $line; + my ($key, $type, $val) = split " ", $line; + # skip blank lines + next if !$key; + # we're done when we find END + last if $key eq "END"; + # skip multi and other lines + next if ($type ne "STR"); + + # printf "%-32.32s %s\n", $key, $val; + + &my_die("no value found for file rule $key in $resolved", $dist_id, $component, + $PS_EXIT_CONFIG_ERROR) if (!$val); + + $file_list{key} = $val; + } + close $in; + } else { + # did not find FILES.OUTPUT in the config file, assume we have an old style file + # and build the distro from the static list which is at the end of this script my $clean_mdc = get_legacy_file_mdc(); my $mdlist = $mdcParser->parse($clean_mdc) or - &my_die("failed to parse clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR); + &my_die("failed to parse legacy file mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR); my $product_lists = parse_md_list($mdlist) or @@ -268,16 +430,14 @@ foreach my $rule (keys %$product_list) { next if $rule eq "STAGE"; - my $keep_on_clean = $product_list->{$rule}; my $fn = $ipprc->filename($rule, $path_base, $component) or &my_die("Missing entry from camera config: $rule", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); - # printf "%-16.16s\t%s\t%s\n", $rule, $clean, $fn; - if (!$clean or $keep_on_clean) { - push @file_list, $fn; - } - } - } - - return \@file_list; + # printf "%-16.16s\t%s\t%s\n", $rule, $clean, $fn; + + $file_list{$rule} = $fn; + } + } + + return \%file_list; }