Changeset 24244 for branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl

Timestamp:

May 26, 2009, 1:59:32 PM (17 years ago)

Author:

beaumont

Message:

merged with trunk

Location:

branches/cnb_branches/cnb_branch_20090301

Files:

• . (modified) (1 prop)
• ippScripts/scripts/dist_component.pl (modified) (12 diffs)

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl

@@ -27 +27 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+
+# filerules that are not included in 'cleaned' distribution bundles
+# I could simplify the function below by using a two level hash but
+# that would be less clear I think
+my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image',
+                     'PPIMAGE.CHIP.MASK' => 'mask',
+                     'PPIMAGE.CHIP.VARIANCE' => 'variance' );
+my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
+my %fake_cleaned;
+my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image',
+                     'PSWARP.OUTPUT.MASK' => 'mask',
+                     'PSWARP.OUTPUT.VARIANCE' => 'variance' );
+my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
+                     'PPSUB.OUTPUT.MASK' => 'mask',
+                     'PPSUB.OUTPUT.VARIANCE' => 'variance' );
+my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
+                      'PPSTACK.OUTPUT.MASK' => 'mask',
+                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
+
 
 # Look for programs we need
@@ -39 +58 @@
 # Parse the command-line arguments
 my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
-my ($outroot, $run_state, $data_state, $magicked);
+my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -52 +71 @@
            'state=s'        => \$run_state,  # state of the run
            'data_state=s'   => \$data_state, # data_state for this component
+           'poor_quality'   => \$poor_quality,  # the processing for this component did not produced images
+           'no_magic'       => \$no_magic,   # magic is not required for this distribution run
            'magicked'       => \$magicked,   # magicked state for this component
-           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'outdir=s'       => \$outdir,     # "directory" for outputs
            'clean'          => \$clean,      # create clean distribution
            'save-temps'     => \$save_temps, # Save temporary files?
@@ -63 +84 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot",
+pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outdir",
            -exitval => 3) unless
     defined $dist_id and
@@ -71 +92 @@
     defined $component and
     defined $path_base and
-    defined $outroot;
+    defined $outdir;
 
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (($stage eq 'raw') and !defined $chip_path_base) {
+if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
     pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
 }
@@ -83 +104 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $basename = basename($path_base);
-
 # making a clean bundle of raw images doesn't make sense
-if (($stage eq "raw") and $clean) {
-    # well I suppose we could ship the log file....
-    &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
-}
 
 # create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outroot, "neb://") != 0) {
-    if (! -e $outroot ) {
-        my $code = system "mkdir -p $outroot";
-        &my_die("cannot create output directory $outroot", $dist_id, $component,
+if (index($outdir, "neb://") != 0) {
+    if (! -e $outdir ) {
+        my $code = system "mkdir -p $outdir";
+        &my_die("cannot create output directory $outdir", $dist_id, $component,
                 $code >> 8) if $code;
     }
 }
 
-my $file_list = get_file_list($clean, $stage, $path_base);
-
-&my_die("failed to compute product list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$file_list;
-
-&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !scalar @$file_list;
-
+# Get the list of data products for this component
+# note: We my_die in get_file_list if something goes wrong. 
+
+my $file_list = get_file_list($stage, $component, $path_base, $clean);
+
+if (($stage ne 'raw') and ($stage ne 'fake')) {
+    # If the file list is empty it is an error because we should at least get a config dump file
+    # except for fake stage which doesn't do anything yet
+    &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
+}
 
 # set up directory for temporary files
 
-my $tmpdir  = "$outroot/tmpdir.$dist_id.$component.$$";
+my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
 if (-e $tmpdir) {
     if (-d $tmpdir) {
@@ -121 +140 @@
                        $PS_EXIT_UNKNOWN_ERROR);
 
-my @base_list;
+#
+# we need to run set masked pixels to NAN in the image and variance images
+# unless
+#   1. we are building a clean distribution bundle
+#   2. magic is not required for this distRun
+#   3. the processing for the component produced no images (warp or diff with bad quality for example)
+my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
+
+my ($image, $mask, $variance);
+
+# foreach my $file_rule (keys %$file_list) {
+my $num_files = 0;
 foreach my $file (@$file_list) {
-    my $base = basename($file);
-    my $path = $ipprc->file_resolve($file);
-    # if file is not found, just skip it.
-    # This is not always the right thing to do.
-    # For example if warpSkyfile is ignored, there won't be many of the data products
-    # the new config dump will only list the files actually produced.
-    next if !$path;
-
-    push @base_list, $base;
-    symlink $path, "$tmpdir/$base";
-}
-
-if (!$clean) {
-
-    # Here is where we determine whether or not a GPC1 image can be released.
-
-    if ($camera eq "GPC1" and !$magicked) {
+    # check whether this file rule refers to an image, mask, or variance fits image
+    my $file_rule = $file->{file_rule};
+    my $image_type = get_image_type($stage, $file_rule);
+
+    # if we are building a clean bundle skip this rule
+    next if $clean && $image_type;
+
+    # if magic is required, don't ship jpegs or binned fits images
+    next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
+            or ($file_rule =~ /.JPEG2/));
+
+    my $file_name = $file->{name};
+    my $base = basename($file_name);
+    my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path) {
+        # skip this file if $poor_quality
+        # this is for compatability with older runs which don't have the files list in the
+        # config dump.
+        # Once we give up on supporting that we can remove the next line. (If the file is in the list
+        # it must exist)
+        next if $poor_quality;
+
+        &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
+    }
+
+    # open the file to make sure it exists (and to work around the failed mount phenomena)
+    my $fh = open_with_retries($path);
+    close $fh;
+
+    # we need to pre-process the image before adding to the bundle. Save the path names.
+    # the images will be created below
+    $num_files++;
+    if ($image_type && $nan_masked_pixels) {
+        # save the 
+        if ($image_type eq 'image') {
+            $image = $path
+        } elsif ($image_type eq 'mask') {
+            $mask = $path;
+        } elsif ($image_type eq 'variance') {
+            $variance = $path;
+        } else {
+            &my_die("invalid image type found: $image_type", $dist_id, $component,
+                       $PS_EXIT_PROG_ERROR);
+        }
+    } else {
+        # create a symbolic link from the file in the nebulous repository
+        # in the temporary directory
+        symlink $path, "$tmpdir/$base";
+    }
+}
+
+if ($nan_masked_pixels) {
+    # One last check as to whether magic has been applied to the inputs
+
+    # Note: the sql for disttool -pendingcomponent won't select a component that 
+    # requires magic and hasn't been magicked, but we check again here
+
+    if (!($magicked or $no_magic)) {
         &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
     }
 
+    &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
+    &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
+    &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+
+    my $class_id;
     # run streaksrelease to set masked pixels to NAN
-    my ($image, $mask, $variance, $class_id);
     if ($stage eq "raw") {
         $class_id = $component;
-        $image = $path_base;
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $class_id);
+        # we can use the chip mask because disttool demands that magic have been run
+        # and so the camera mask and the chip mask are the same
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
+        my $fh = open_with_retries($mask);
+        close $fh;
     } elsif ($stage eq "chip") {
         $class_id = $component;
-        $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
-        $variance = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-    } elsif ($stage eq "warp") {
-        $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
-        $variance = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
-    } elsif ($stage eq "diff") {
-        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
-        $variance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
-    } else {
-        &my_die("not ready for stage: $stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
-    }
-
-    push @base_list, $image;
-    push @base_list, $mask if $mask;
-    push @base_list, $variance if $variance;
+    }
 
     my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
@@ -185 +249 @@
 my $bytes;
 my $md5sum;
-# create the tarfile
-{
+
+if ($num_files) {
+    # create the tarfile
     # XXX TODO: create a file rule for the tar file name
     my $tbase = basename($path_base);
     $tbase .= ".$component" if $component;
     $file_name = "$tbase.tgz";
-    my $tarfile = "$outroot/$file_name";
-
-    my $command = "tar -C $tmpdir -czhf $tarfile .";
+    my $tarfile = "$outdir/$file_name";
+
+    my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -207 +272 @@
     &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
 
-    my @finfo = stat($tarfile);
-    &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo;
-    $bytes = $finfo[7];
+    $bytes = -s $tarfile;
 
     delete_tmpdir($tmpdir);
+} else {
+    # no files for this component
+    $file_name = "none";
+    $bytes = 0;
+    $md5sum = "0";
 }
 {
@@ -234 +302 @@
 ### Pau.
 
+# return the image type (image, mask, or variance) if this file rule refers to 
+# one of the big fits files that are not included in a clean distribution
+sub get_image_type {
+    my $stage = shift;
+    my $rule = shift;
+    my $type;
+
+    if ($stage eq "raw") {
+        if ($rule eq "RAW.IMAGE") {
+            $type = 'image';
+        }
+    } elsif ($stage eq "chip") {
+        $type = $chip_cleaned{$rule};
+    } elsif ($stage eq "camera") {
+        $type = $camera_cleaned{$rule};
+    } elsif ($stage eq "fake") {
+        $type = $fake_cleaned{$rule};
+    } elsif ($stage eq "warp") {
+        $type = $warp_cleaned{$rule};
+    } elsif ($stage eq "diff") {
+        $type = $diff_cleaned{$rule};
+    } elsif ($stage eq "stack") {
+        $type = $stack_cleaned{$rule};
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    return $type;
+}
+
+sub open_with_retries {
+    my $name = shift;
+    
+    my $tries = 1;
+    my $max_tries = 5;
+    my $opened = 0;
+    while (!$opened && ($tries <= $max_tries)) {
+        $opened = open(IN, "<$name");
+        if (!$opened) {
+            print STDERR "WARNING failed to open $name re-try $tries\n";
+            $tries++;
+            sleep 5;
+        }
+    }
+    
+    &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
+                    $PS_EXIT_DATA_ERROR) if (!$opened);
+
+    return *IN;
+}
 
 sub get_file_list {
+    my $stage = shift;
+    my $component = shift;
+    my $path_base = shift;
     my $clean = shift;
-    my $stage = shift;
-    my $path_base = shift;
 
     my @file_list;
     if ($stage eq "raw") {
-        push @file_list, $path_base;
-    } else {
-        # TODO: these data will eventually come from the CONFIG dump
-
-        my $clean_mdc = get_legacy_file_mdc();
-
-        my $mdlist = $mdcParser->parse($clean_mdc) or
-                &my_die("failed to parse clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
-
-        my $product_lists = parse_md_list($mdlist) or
-                &my_die("failed to parse metadata list", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
-
-
-        my $product_list;
-        foreach my $list (@$product_lists) {
-
-            my $list_name = $list->{STAGE};
-            if (lc ($list_name) eq lc($stage) ) {
-                $product_list = $list;
-            }
-        }
-        &my_die("failed to parse find product list for stage $stage", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$product_list;
-
-        # resolve each file rule and add it to the file_list
-        foreach my $rule (keys %$product_list) {
-            next if $rule eq "STAGE";
-            my $keep_on_clean = $product_list->{$rule};
-            my $fn = $ipprc->filename($rule, $path_base, $component) or
-                &my_die("Missing entry from camera config: $rule", $dist_id, $component,
+        # XXX: TODO for now disttool sets path_base is set to the uri of the file
+        # eventually rawImfile will have a path_base that we'll need to add '.fits'
+        # XXX do we want to distribute the registration log files?
+        if (!$clean) {
+            my %file;
+            $file{file_rule} = "RAW.IMAGE";
+            $file{name} = $path_base;
+            push @file_list, \%file;
+        }
+        return \@file_list;
+    }
+
+    # we get the list of output data products for this run from the config dump file that
+    # is created when the run is done
+    my $config_file_rule;
+    if ($stage eq "chip") {
+        $config_file_rule = "PPIMAGE.CONFIG";
+    } elsif ($stage eq "camera") {
+        $config_file_rule = "PSASTRO.CONFIG";
+    } elsif ($stage eq 'fake') {
+        # XXX: fake is a no op now return an emtpy list
+        return \@file_list;
+    } elsif ($stage eq "warp") {
+        $config_file_rule = "PSWARP.CONFIG";
+    } elsif ($stage eq "diff") {
+        $config_file_rule = "PPSUB.CONFIG";
+    } elsif ($stage eq "stack") {
+        $config_file_rule = "PPSTACK.CONFIG";
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
+                &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
                     $PS_EXIT_CONFIG_ERROR);
-    #        printf "%-16.16s\t%s\t%s\n",  $rule, $clean, $fn;
-            if (!$clean or $keep_on_clean) {
-                push @file_list, $fn;
-            }
-        }
-    }
+
+    my %config_file_hash;
+
+    # add the configuration file to the list
+    $config_file_hash{file_rule} = $config_file_rule;
+    $config_file_hash{name} = $config_file;
+    push @file_list, \%config_file_hash;
+
+    my $resolved = $ipprc->file_resolve($config_file);
+
+    &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$resolved);
+
+    # we don't use the mdc parser because the perl parser is way is too slow for complicated config
+    # files like this
+    my $in = open_with_retries($resolved);
+
+    my $line;
+    while ($line = <$in>) {
+        if ($line =~ /FILES.OUTPUT/) {
+            # print "found FILES.OUTPUT\n";
+            last;
+        }
+    }
+    &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$line);
+
+    while ($line = <$in>) {
+        chomp $line;
+        my ($key, $type, $val) = split " ", $line;
+        # skip blank lines
+        next if !$key;
+        # we're done when we find END
+        last if $key eq "END";
+        # skip multi and other lines
+        next if ($type ne "STR");
+
+        # printf "%-32.32s   %s\n", $key, $val;
+
+        &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
+                $PS_EXIT_CONFIG_ERROR) if (!$val);
+
+        my %file;
+        $file{file_rule} = $key;
+        $file{name} = $val;
+        push @file_list, \%file;
+    }
+    close $in;
 
     return \@file_list;
@@ -304 +468 @@
     $command   .= " -dist_id $dist_id";
     $command   .= " -component $component";
-    $command   .= " -code $exit_code";
+    $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -326 +490 @@
 sub delete_tmpdir
 {
-    if (!$save_temps) {
+    if (!$save_temps and $tmpdir and -e $tmpdir) {
         system "rm -r $tmpdir";
     }
 }
 
-# list of output data products for runs that were made before the configuration re-work 
-sub get_legacy_file_mdc
-{
-    my $list =
-"
-#
-# interesting things we might want to save in this file
-#   1. whether file is to be distributed in 'clean' mode
-#   2. whether file is to be cleaned (this should be the same thing)
-#   3. type of file 'no, not our business, precious'
-
-# this data should probably be computed by the relevant program and saved as part
-# of the configuration dump that is output.
-
-
-PROD_LIST MULTI
-
-PROD_LIST   METADATA
-        STAGE                    STR     RAW
-    # there is isn't really a file rule for raw images
-        DUMMY                   BOOL    F
-END
-
-# list of data products for a gpc1 chipProcessedImfile  (made by ppImage)
-PROD_LIST   METADATA
-        STAGE                    STR     CHIP
-        PPIMAGE.CONFIG          BOOL    T
-#       PPIMAGE.CHIP            BOOL    F
-#       PPIMAGE.CHIP.MASK       BOOL    F
-#       PPIMAGE.CHIP.VARIANCE   BOOL    F
-        PPIMAGE.BIN1            BOOL    T
-        PPIMAGE.BIN2            BOOL    T
-        PSPHOT.OUT.CMF.SPL      BOOL    T
-        PSPHOT.BACKMDL          BOOL    T
-        PPIMAGE.STATS           BOOL    T
-        LOG.IMFILE              BOOL    T
-        TRACE.IMFILE            BOOL    T
-    # where do we put exposure level data such as LOG.EXP ?
-END
-# exposure level output products from camera processing made by psastro and ppImage (jpegs)
-PROD_LIST   METADATA
-        STAGE                    STR     CAM
-        PSASTRO.CONFIG          BOOL    T
-        PSASTRO.OUTPUT          BOOL    T
-        PSASTRO.STATS           BOOL    T
-        PPIMAGE.JPEG1           BOOL    T
-        PPIMAGE.JPEG2           BOOL    T
-        LOG.EXP                 BOOL    T
-        TRACE.EXP               BOOL    T
-END
-# chip lelel output products from camera processing made by psastro
-PROD_LIST   METADATA
-        STAGE                    STR CAM_CHIP
-#       PSASTRO.OUTPUT.MASK     BOOL    F
-END
-PROD_LIST   METADATA
-        STAGE                    STR     FAKE
-#       PPSIM.OUTPUT            BOOL    F
-END
-# list of data products for a gpc1 warpSkyfile (pswarp)
-PROD_LIST   METADATA
-        STAGE                    STR     WARP
-        PSWARP.CONFIG           BOOL    T
-#       PSWARP.OUTPUT           BOOL    F
-#       PSWARP.OUTPUT.MASK      BOOL    F
-#       PSWARP.OUTPUT.VARIANCE  BOOL    F
-        PSWARP.OUTPUT.SOURCES   BOOL    T
-        PSPHOT.BACKMDL.MEF      BOOL    T
-        PSPHOT.PSF.SKY.SAVE     BOOL    T
-        SKYCELL.STATS           BOOL    T
-        SKYCELL.TEMPLATE        BOOL    T
-        LOG.EXP                 BOOL    T
-        TRACE.EXP               BOOL    T
-END
-# outputs from diffRun (ppSub)
-PROD_LIST   METADATA
-        STAGE                    STR     DIFF
-        PPSUB.CONFIG            BOOL    T
-#       PPSUB.OUTPUT            BOOL    F
-#       PPSUB.OUTPUT.MASK       BOOL    F
-#       PPSUB.OUTPUT.VARIANCE   BOOL    F
-        PPSUB.OUTPUT.KERNELS    BOOL    T
-        PPSUB.OUTPUT.JPEG1      BOOL    T
-        PPSUB.OUTPUT.JPEG2      BOOL    T
-        PSPHOT.OUT.CMF.MEF      BOOL    T
-        PSPHOT.BACKMDL.MEF      BOOL    T
-        SKYCELL.STATS           BOOL    T
-        LOG.EXP                 BOOL    T
-        TRACE.EXP               BOOL    T
-END
-PROD_LIST   METADATA
-        STAGE                    STR     STACK
-        PPSTACK.CONFIG          BOOL    T
-#       PPSTACK.OUTPUT          BOOL    F
-#       PPSTACK.OUTPUT.MASK     BOOL    F
-#       PPSTACK.OUTPUT.VARIANCE BOOL    F
-        PPSTACK.TARGET.PSF      BOOL    T
-        PSPHOT.OUT.CMF.MEF      BOOL    T
-        PSPHOT.BACKMDL.MEF      BOOL    T
-        SKYCELL.STATS           BOOL    T
-        PPSTACK.OUTPUT.JPEG1    BOOL    T
-        PPSTACK.OUTPUT.JPEG2    BOOL    T
-        LOG.EXP                 BOOL    T
-        TRACE.EXP               BOOL    T
-END
-";
-    return $list;
-}
-
-
 __END__