Changeset 30118 for branches/czw_branch/20101203/ippScripts

Timestamp:

Dec 20, 2010, 2:30:45 PM (16 years ago)

Author:

watersc1

Message:

Attempting to bring branch in sync with trunk

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippScripts (modified) (1 prop)
• ippScripts/scripts/automate_stacks.pl (modified) (2 diffs)
• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/chip_imfile.pl (modified) (9 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (2 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (1 diff)
• ippScripts/scripts/magic_destreak_defineruns.pl (modified) (1 diff)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (3 diffs)
• ippScripts/scripts/publish_file.pl (modified) (5 diffs)
• ippScripts/scripts/receive_file.pl (modified) (2 diffs)
• ippScripts/scripts/register_imfile.pl (modified) (2 diffs)

branches/czw_branch/20101203/ippScripts/scripts/automate_stacks.pl

@@ -1355 +1355 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    if ($target eq 'OSS') {
+    if (($target eq 'OSS')||($target eq 'SweetSpot')) {
         my $db = init_gpc_db();
 
@@ -1372 +1372 @@
             
             if (($#{ $warps } + 1) % 2 != 0) {
-                print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
-                last;
+                print STDERR "Number of input warps to make OSS diffs is not even! $this_object $#{ $warps }\n";
+#               last;
+                next;
             }
             

branches/czw_branch/20101203/ippScripts/scripts/camera_exp.pl

@@ -378 +378 @@
         $ipprc->file_exists($file);
 
-    if ($replicate and (file_scheme($file) eq 'neb')) {
+    my $scheme = file_scheme($file);
+    if ($replicate and $scheme and (file_scheme($file) eq 'neb')) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $cam_id, $PS_EXIT_SYS_ERROR);
     }

branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl

@@ -159 +159 @@
         if (storage_object_exists($configuration, \$gone)) {
             if ($gone) {
-                rename_gone_file($configuration);
                 $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
                 # if we dump the config we need to insure that the config dump represents
@@ -177 +176 @@
     $ipprc->delete_destreak_backup_file($outputWeight)
         or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # don't do binned images when updating unless we are starting from scratch.
+    $do_binned_images = 0 unless $dump_config;
 }
 if ($do_binned_images) {
@@ -491 +493 @@
         check_output($configuration, 1) if $dump_config;
         check_output($backmdl, 1) if $outputBackmdlExpect;
-        check_output($pattern, 1) if $outputPatternExpect;
+        # allow the pattern file to be missing if run state is update older data doesn't have one
+        # I should parse the config dump file to calculate the 'Expect' variables
+        check_output($pattern, 1, $run_state eq 'update') if $outputPatternExpect;
         if ($do_photom) {
             check_output($outputSources, 1);
@@ -556 +560 @@
             if ($gone) {
                 carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
-                rename_gone_file($outputSources);
                 $make_sources = 1;
             }
@@ -579 +582 @@
             if ($gone) {
                 carp "WARNING: PSF storage object exists but all instances are permanently gone";
-                rename_gone_file($outputPsf);
                 $make_psf = 1;
             }
@@ -613 +615 @@
 sub storage_object_exists
 {
+    return 0 if !$neb;
+
     my $file = shift;
     my $ref_all_gone = shift;
+
 
     my $exists = $neb->storage_object_exists($file);
@@ -678 +683 @@
     return 1;
 }
-sub rename_gone_file
-{
-    # check whether the only instance of file is on a lost volume
-    # XXX: we don't have a proper interface for this. 
-    # For now try to use Bill's hack: the script 'whichnode'
-
-    my $file = shift;
-    $neb->move($file, "$file.gone") or 
-                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-}
 
 # Prepare to write to an output file
@@ -711 +706 @@
     my $file = shift;
     my $replicate = shift;
+    my $allow_missing = shift;
 
     if (!defined $file) {
@@ -716 +712 @@
     }
 
-    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
-
-    if ($replicate and (file_scheme($file) eq 'neb')) {
+    my $exists = $ipprc->file_exists($file);
+
+    if (!$exists) {
+        if ($allow_missing) {
+            carp("Couldn't find expected output_file: $file but continuing anyways\n");
+            return 1;
+        }
+        &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+
+    if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     }

branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl

@@ -62 +62 @@
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $file_cmd   = can_run('file') or (warn "can't find program file" and $missing_tools = 1);
+my $zcat   = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -526 +528 @@
                     $PS_EXIT_CONFIG_ERROR) if (!$resolved);
 
+    &my_die("config dump file resolved but not accessible: $config_file_rule", $component,
+                    $PS_EXIT_CONFIG_ERROR) if !$ipprc->file_exists($resolved);
+
+    my $mdc_compressed;
+    {
+        my $command = "$file_cmd $resolved";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        my $output = join "", @$stdout_buf;
+        # XXX: may need to to make this more robust
+        $mdc_compressed = ($output =~ /gzip/);
+    }
+    my $inName;
+    if ($mdc_compressed) {
+        my $tmpfile;
+        ($tmpfile, $inName) = tempfile( "/tmp/bundle.XXXX", UNLINK => !$save_temps );
+        close($tmpfile) or &my_die("failed to close $inName", $component, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $command = "$zcat $resolved > $inName";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+
+
+    } else {
+        $inName = $resolved;
+    }
+    my $in = open_with_retries($inName);
+
     # we don't use the mdc parser because the perl parser is way is too slow for complicated config
     # files like this
-    my $in = open_with_retries($resolved);
-
     my $line;
     while ($line = <$in>) {

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl

@@ -309 +309 @@
         }
 
-        $sources = $ipprc->filename("PSPHOT.OUTPUT",  $path_base, $class_id);
+        # only destreak cmf file if this is a new run
+        if ($run_state eq 'new') {
+            $sources = $ipprc->filename("PSPHOT.OUTPUT",  $path_base, $class_id);
+        }
 
     } elsif ($stage eq "chip_bg") {

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl

@@ -78 +78 @@
     foreach my $label (@labels) {
         my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";

branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl

@@ -191 +191 @@
     $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
     $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-    $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id);
 
     # we use the mask output from the camera stage for input and replace
@@ -204 +203 @@
     }
 
+    # only revert sources if this is a new run, not one being updated
+    if ($run_state eq 'new') {
+        $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id);
+        $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id);
+    }
+
     $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id);
     # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
@@ -209 +214 @@
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
-    $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id);
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);

branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl

@@ -44 +44 @@
 my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
 my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
+my ( $output_format );
 
 GetOptions(
@@ -59 +60 @@
     'save-temps'        => \$save_temps, # Save temporary files?
     'redirect-output'   => \$redirect,   # Redirect output to log file?
+    'output_format=i'   => \$output_format, # Output format for ppMops
     ) or pod2usage( 2 );
 
@@ -191 +193 @@
                      warp_id => $comp->{warp1},
                      diff_id => $comp->{diff_id},
+                     output_format => $comp->{output_format},
                      direction => 1,
         };
@@ -323 +326 @@
 
     my $command = "$ppMops $input $output";
-    $command .= " -version 1"; # XXX : NOTE: for now, MOPS just wants PS1_DV1 (remove this eventually...)
     $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
     $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
@@ -335 +337 @@
     $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
     $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
+    $command .= " -version " . $data->{output_format} if defined $data->{output_format};
 
     unless ($no_op) {

branches/czw_branch/20101203/ippScripts/scripts/receive_file.pl

@@ -32 +32 @@
 my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
 my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1);
+my $gunzip = can_run('gunzip') or (warn "Can't find gunzip" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -404 +405 @@
     my $workdir = shift;
 
+    my $filecmd_output = `file $src`;
+    &my_die("failed to determinte file type of $src", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$filecmd_output;
+    chomp $filecmd_output;
+
+    my $tmpName;
+    if ($filecmd_output =~ /gzip/) {
+        my $tmpfile;
+        ($tmpfile, $tmpName) = tempfile( "/tmp/receive.XXXX", UNLINK => !$save_temps );
+        close($tmpfile) or &my_die("failed to close $tmpName", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $command = "$gunzip -c $src > $tmpName";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        $src = $tmpName;
+    }
+
     open my $IN,  "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR);
     open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR);

branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl

@@ -287 +287 @@
     if ($burntool_data->{burnable} == 0) {
         $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+                exit($error_code);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
     }
     else {
@@ -296 +307 @@
         $apply_command .= " --verbose " if $verbose;
         print "$apply_command\n";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            IPC::Cmd::run(command => $apply_command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
-            exit($error_code);
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
+                exit($error_code);
+            }
         }
     }    
 
-    unless ($no_update) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn ("Unable to perform regtool -addprocessedimfile: $error_code");
-            exit($error_code);
-        }
-    } else {
-        print "skipping command: $command\n";
-    }
 
 }