Changeset 36458

Timestamp:

Jan 21, 2014, 10:29:30 AM (12 years ago)

Author:

bills

Message:

from trunk. Deal with missing convovled stack images, compressed log files, and
some changes for stack_summary stage.

File:

• tags/ipp-20130712/ippScripts/scripts/dist_bundle.pl (modified) (10 diffs, 1 prop)

tags/ipp-20130712/ippScripts/scripts/dist_bundle.pl

@@ -50 +50 @@
                      'PPSUB.INVERSE.MASK' => 'inv_mask',
                      'PPSUB.INVERSE.VARIANCE' => 'inv_variance' );
-my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
+
+my %stack_cleaned = ( 'PPSTACK.UNCONV' => 'image',
+                      'PPSTACK.UNCONV.MASK' => 'mask',
+                      'PPSTACK.UNCONV.VARIANCE' => 'variance',
+                      'PPSTACK.UNCONV.EXP' => 'exp',
+                      'PPSTACK.UNCONV.EXPNUM' => 'expnum',
+                      'PPSTACK.UNCONV.EXPWT' => 'expwt'
+                      );
+
+my %stack_convolved = ( 'PPSTACK.OUTPUT' => 'image',
                       'PPSTACK.OUTPUT.MASK' => 'mask',
-                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
-my %sky_cleaned   = ( 'PSPHOT.STACK.OUTPUT.IMAGE' => 'image',
-                      'PSPHOT.STACK.OUTPUT.MASK' => 'mask',
-                      'PSPHOT.STACK.OUTPUT.VARIANCE' => 'variance' );
+                      'PPSTACK.OUTPUT.VARIANCE' => 'variance',
+                      'PPSTACK.OUTPUT.EXP' => 'exp',
+                      'PPSTACK.OUTPUT.EXPNUM' => 'expnum',
+                      'PPSTACK.OUTPUT.EXPWT' => 'expwt'
+                      );
+my %empty_cleaned = ();
+
 
 
@@ -110 +122 @@
     defined $outroot;
 
+$no_magic = 1;
+
 if ($stage eq 'raw' and !$clean and !$no_magic) {
     
@@ -134 +148 @@
 my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs);
 
-if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) {
+if (($stage ne 'raw') and ($stage ne 'fake') and ($stage ne 'stack_summary') and !$poor_quality) {
     # If the file list is empty it is an error because we should at least get a config dump file
-    # except for fake stage which doesn't do anything yet
+    # except for fake stage which doesn't do anything yet, stack_summary stage, and of course raw files have
+    # no config dump because they aren't processed
     &my_die("empty file list", $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
 }
@@ -165 +180 @@
     my $image_type = get_image_type($stage, $file_rule);
 
+    # skip useless empty trace files
+    next if ($file_rule =~ /TRACE/);
+
     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
     next if $image_type and ($clean or $poor_quality);
+
+    if ($stage eq 'stack') {
+        # skip convolved stacks since they are deleted just after they are created now
+        next if $stack_convolved{$file_rule};
+    }
 
     # if magic is required, don't ship jpegs or binned fits images
@@ -180 +203 @@
 
     my $file_name = $file->{name};
-    my $base = basename($file_name);
     my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path and $file_rule =~ /LOG/) {
+        my $compressed_file_name = $file_name . '.bz2';
+        my $compressed_path = $ipprc->file_resolve($compressed_file_name);
+        if ($compressed_path) {
+            $path = $compressed_path;
+            $file_name = $compressed_file_name;
+        }
+    }
+
 
     if (!$path) {
@@ -194 +226 @@
         # XXX: perhaps only do this for stages where we know that this happens
         next if $file_rule =~ /STATS/;
-        # don't fail on these "non-essential files"
+        # don't fail if these "non-essential files" are missing
+        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /TRACE/;
-        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /BIN/;
 
-        # don't fail if cmf file is missing they are not regenerated on update
-        next if ($stage eq 'chip' and $file_rule eq 'PSPHOT.OUTPUT');
+        # chip stage cmf files are not regenerated after cleanup so don't complain if
+        # they are missing. They are of limited utility anyways. USE the smfs
+        next if ($stage eq 'chip' and $file_rule eq 'PSPHOT.OUTPUT');
 
         &my_die("failed to resolve  $file_name", $component, $PS_EXIT_DATA_ERROR);
@@ -235 +268 @@
         # create a symbolic link from the file in the nebulous repository
         # in the temporary directory
-        symlink $path, "$tmpdir/$base";
+        symlink $path, "$tmpdir/" . basename($file_name);
     }
 }
@@ -463 +496 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky") {
-        $type = $sky_cleaned{$rule};
+    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') {
+        $type = $empty_cleaned{$rule};
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -549 +582 @@
         $config_file_rule = "PPSTACK.CONFIG";
     } elsif ($stage eq "sky") {
-        if (1 or $num_sky_inputs > 1) {
-            $config_file_rule = "PSPHOT.STACK.CONFIG";
-        } else {
-            $config_file_rule = "PSPHOT.SKY.CONFIG";
-        }
+        $config_file_rule = "PSPHOT.STACK.CONFIG";
+    } elsif ($stage eq "skycal") {
+        $config_file_rule = "PSASTRO.CONFIG";
+    } elsif ($stage eq "stack_summary") {
+        # stack_summary stage does not use a config dump file.
+        return build_stack_summary_file_list($path_base, \@file_list);
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -668 +702 @@
     if (!$num_inputs) {
         $num_inputs = "undefined" if !defined $num_inputs;
-        &my_die("unexpected number of static sky inputs $num_inputs",  $PS_EXIT_PROG_ERROR, $component, $error_code);
+        &my_die("unexpected number of static sky inputs $num_inputs",  $component, $error_code);
     }
 
     return $num_inputs;
+}
+
+# For stack_summary stage, we don't have a config dump file to give us the list of files.
+# For now run neb-ls on the path base and take everything that matches.
+sub build_stack_summary_file_list {
+    my ($path_base, $file_list) = @_;
+
+    my $scheme = file_scheme($path_base);
+
+    if (!$scheme or $scheme ne 'neb') {
+        &my_die("Building bundles for stack_summary not supported for no nebulous path_base.",
+            $component, $PS_EXIT_PROG_ERROR);
+    }
+
+    my $command = "neb-ls $path_base" . '%';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $component, $error_code);
+    }
+    my $nebls_output = join "", @$stdout_buf;
+
+    my @files = split "\n", $nebls_output;
+
+    my $i = 0;
+    foreach my $f (@files) {
+        my %file;
+        $file{file_rule} = sprintf 'RULE.%d', $i++;
+        $file{name} = 'neb://any/' . $f;
+        push @$file_list, \%file;
+    }
+
+    return $file_list;
 }