Changeset 33415 for branches/meh_branches/ppstack_test/ippScripts/scripts/chip_imfile.pl

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippScripts/scripts/chip_imfile.pl (modified) (14 diffs)

branches/meh_branches/ppstack_test/ippScripts/scripts/chip_imfile.pl

@@ -34 +34 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
+
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -44 +46 @@
 my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose,
      $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned );
-
-my $zaplog = 0;
 
 GetOptions(
@@ -65 +65 @@
     'no-op'             => \$no_op,     # Don't do any operations?
     'redirect-output'   => \$redirect,
-    'zaplog'            => \$zaplog,
     'save-temps'        => \$save_temps, # Save temporary files?
     ) or pod2usage( 2 );
@@ -92 +91 @@
 }
 
-my ($logDest, $traceDest);
+my ($logRule, $traceDest);
 if ($run_state eq 'new') {
-    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog);
+    $logRule = "LOG.IMFILE";
     $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
 } else {
-    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
+    $logRule = "LOG.IMFILE.UPDATE";
     $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
 }
 
 if ($redirect) {
-    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+    my $logDest = $ipprc->filename($logRule, $outroot, $class_id);
+
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+
     print STDOUT "\n\n";
     print STDOUT "Starting script $0 on $host\n\n";
@@ -145 +147 @@
 my $outputBin2;
 my $dump_config = 1;
+my $do_stats = 1;
+$outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
 my $do_binned_images = 1;
 if ($run_state eq 'new') {
     # prepare the files that are only created for a new run
     $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
-    $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
 } else {
     $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) 
@@ -194 +197 @@
 unless ($no_op) {
     my $command;
-    my $do_stats;
 
     ## get the ppImage recipe for this camera and CHIP reduction
@@ -266 +268 @@
         }
 
-        ## Read camera config to get the current good burntool state
-        ## this returns just the values of interest in a tiny mdc file
-        my $camera_config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD -get-key BURNTOOL.STATE.GOOD.UPDATE";
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $camera_config_cmd, verbose => 0);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-        }
-
-        my $camData = $mdcParser->parse(join "", @$stdout_buf) or
-            &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-
-        my $burntoolStateGood;
-        my $burntoolStateGoodUpdate;
-        foreach my $camEntry (@$camData) {
-            if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") {
-                $burntoolStateGood = $camEntry->{value};
-            }
-            if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD.UPDATE") {
-                $burntoolStateGoodUpdate = $camEntry->{value};
-            }
-        }
+        ## Read camera config to get the current good burntool state :
+        ## XXX This is extremely slow. Any better way to do this?
+        ## my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -";
+        ## ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        ##     run(command => $camera_config_cmd, verbose => 0);
+        ## unless ($success) {
+        ##     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        ##     &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        ## }
+        ##
+        ## my $camData = $mdcParser->parse(join "", @$stdout_buf) or
+        ##     &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+        ## XXX short term hack until we have a C-based parser
+        my $burntoolStateGood = 14;
+        my $burntoolStateGoodUpdate = 13;
+        ## foreach my $camEntry (@$camData) {
+        ##     if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") {
+        ##         $burntoolStateGood = $camEntry->{value};
+        ##     }
+        ##     if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD.UPDATE") {
+        ##         $burntoolStateGoodUpdate = $camEntry->{value};
+        ##     }
+        ## }
 
         if ($run_state eq 'new') {
@@ -329 +332 @@
         # get the UNIX version of the (possible) neb: or path: filename
         my $uriReal = $ipprc->file_resolve( $uri );
+        # Catch errors here
+        if (!$uriReal) {
+            my $repair_cmd = "$nebrepair $uri";
+            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+            unless ($repair_success) {
+                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            $uriReal = $ipprc->file_resolve( $uri );
+        }
+            
         &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
@@ -335 +348 @@
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
         unless ($success) {
+            # Catch errors here
+            my $repair_cmd = "$nebrepair $uri";
+            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+            unless ($repair_success) {
+                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+        }
+        unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
@@ -347 +369 @@
             $burntoolTable_uri =~ s/fits$/burn.tbl/;
             $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+                my $repair_cmd = "$nebrepair $burntoolTable_uri";
+                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+                unless ($repair_success) {
+                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                }
+                $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            }
             unless ($ipprc->file_exists($burntoolTable_uri)) {
+                # Catch errors here
                 &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
             }
@@ -439 +470 @@
         $command .= " -Db PPIMAGE:PHOTOM FALSE";
     }
-    if ($run_state eq "new") {
+    if ($run_state eq "new" or $do_stats) {
         $command .= " -recipe PPSTATS CHIPSTATS";
         $command .= " -stats $outputStats";
@@ -450 +481 @@
     $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     $command .= " -source_id $source_id" if defined $source_id;
-    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -tracedest $traceDest";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -531 +562 @@
 
 # Add the processed file to the database
-unless ($no_update) {
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
-        exit($error_code);
-    }
-} else {
-    print "skipping command: $command\n";
+my $tries = 0;
+while (1) {
+    $tries++;
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
+            exit($error_code) if $tries >= 3;
+            warn("Waiting 10 seconds to try again\n");
+            sleep 10;
+        } else {
+            last;
+        }
+    } else {
+        print "skipping command: $command\n";
+        last;
+    }
 }
 
@@ -726 +766 @@
         &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
 
     if ($replicate and $neb) {