Changeset 25425 for trunk/tools

Timestamp:

Sep 16, 2009, 6:35:31 PM (17 years ago)

Author:

watersc1

Message:

changes to support new burntool_state

Location:

trunk/tools

Files:

• examine_burntool_pcontrol.pl (modified) (7 diffs)
• ipp_apply_burntool.pl (modified) (7 diffs)

trunk/tools/examine_burntool_pcontrol.pl

@@ -2 +2 @@
 
 # script to print out information about the current status of a burntool run.
-
 use DBI;
 use Getopt::Std;
@@ -15 +14 @@
     print STDERR "         status of the processing.  On the OTA grid:\n";
     print STDERR "                  _             Blank chip\n";
-    print STDERR "                  N             Null value for user_1. No burntool attempted.\n";
-    print STDERR "                  O             0.1 value for user_1. Burntool failed or is running.\n";
-    print STDERR "                  B             1.0 value for user_1. Burntool succeeded.\n";
+    print STDERR "                  N             Null/zero value for burntool_state. No burntool attempted.\n";
+    print STDERR "                  O             -1 value for burntool_state. Burntool is running.\n";
+    print STDERR "                  E             -2 value for burntool_state. Error!\n";
+    print STDERR "                  B             Current version value for burntool_state. Burntool succeeded.\n";
+    print STDERR "                  b             Old version value for burntool_state. Burntool needs to be rerun.\n";
     exit(1);
 }
-
-# Read a class_id and user_1 pair, and set the correct cell of the (sorry, global) character array vector
+$| = 1;
+# Determine what the value of "BURNTOOL.STATE.GOOD" currently is:
+$burntoolStateGood = 999;
+open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n";
+while(<LAZY>) {
+    chomp;
+    if ($_ =~ /BURNTOOL.STATE.GOOD/) {
+        @line = split /\s+/;
+        $burntoolStateGood = $line[2];
+    }
+}
+close(LAZY);
+unless( $burntoolStateGood == 999) {
+    print STDERR "GOOD == $burntoolStateGood\n";
+}
+
+# Read a class_id and burntool_state pair, and set the correct cell of the (sorry, global) character array vector
 sub class_to_vector {
     my $class = shift;
-    my $user1 = shift;
+    my $burntool_state = shift;
     my $char = '_';
-    if ($user1 == 1.0) {
-        $char = '[32mB[m';
+    if (abs($burntool_state) == $burntoolStateGood) {
+        $char = '[32;01mB[m';
         $burncount++;
     }
-    elsif ($user1 == 0.1) {
+    elsif ($burntool_state == -1) {
+        $char = '[01;33mP[m';
+    }
+    elsif ($burntool_state == 0) {
         $char = '[31mO[m';
     }
-    elsif ($user1 == -1.0) {
-        $char = '[34mN[m';
+    elsif ($burntool_state == -2) {
+        $char = '[35;01;44;05mE[m';
+    }
+    elsif ($burntool_state == -3) {
+        $char = '[34mX[m';
+    }
+    else {
+        $char = '[32mb[m';
     }
     
@@ -47 +72 @@
 # Engauge autoflush, in case the database server is acting up.
 $| = 1;
-
 
 # Load the pcontrol.pro file, and create arrays of the dates included
@@ -96 +120 @@
     # Get the data.  The limit is there to keep the database happy.  I don't think you can observe that many
     # images on a single night.
-    $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,user_1,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 60000";
+    $sth = "SELECT exp_id,exp_name,obs_mode,dateobs,class_id,burntool_state,comment FROM rawImfile WHERE dateobs >= '${dmin}' AND dateobs <= '${dmax}' order by dateobs limit 60000";
     $data_ref = $db->selectall_arrayref( $sth );
 
@@ -122 +146 @@
     
     foreach $row_ref (@{ $data_ref }) {
-        ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$user_1,$comment) = @{ $row_ref };
+        ($exp_id,$exp_name, $obs_mode,$dateobs,$class_id,$burntool_state,$comment) = @{ $row_ref };
+
         # Correct for nulls in the database.
         if (!defined($obs_mode)) {
             $obs_mode = 'NULL';
         }
-        if (!defined($user_1)) {
-            $user_1 = -1;
+        if (!defined($burntool_state)) {
+            $burntool_state = 0;
         }
         if (!defined($comment)) {
             $comment = ' ';
         }
-
+#       print "$exp_id $class_id $burntool_state\n";
         # If we're still working on the same exp_id, update the chip we got
         if ($exp_id == $cur_exp_id) {
-            class_to_vector($class_id,$user_1);
+            class_to_vector($class_id,$burntool_state);
         }
         else {
@@ -156 +181 @@
             }
             ($cur_exp_id,$cur_exp_name,$cur_obs_mode,$cur_dateobs,$cur_comment) = ($exp_id,$exp_name,$obs_mode,$dateobs,$comment);
-            class_to_vector($class_id,$user_1);
+            class_to_vector($class_id,$burntool_state);
         }
     }
@@ -169 +194 @@
     if ($burncount != 60) {
         for ($j = 0; $j < 64; $j++) {
-            $vector[$j] =~ s/\[..m(.).*/$1/;
-            if ($vector[$j] ne 'B') {
+#           printf("%d %s %d\n",$j, $vector[$j], ($vector[$j] !~ /B/));
+            if ($vector[$j] !~ /B/) {
                 if ($vector[$j] ne '_') {
                     $id = sprintf("XY%d%d",int($j / 8),$j % 8);

trunk/tools/ipp_apply_burntool.pl

@@ -12 +12 @@
 use Carp;
 
-use IPC::Cmd 0.36 qw( can_run );
-use IPC::Run 0.36 qw( run );
+use IPC::Cmd 0.36 qw( can_run run );
+#use IPC::Run 0.36 qw( run );
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
@@ -20 +20 @@
 use Pod::Usage qw( pod2usage );
 
-my ( $class_id, $dateobs_begin, $dateobs_end, $skip_burned, $rerun_from_first, $dbname, $logfile, $verbose, $save_temps);
+my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $camera, $dbname, $logfile, $verbose, $save_temps);
 GetOptions(
     'class_id=s'        => \$class_id, # chip identifier
     'dateobs_begin=s'   => \$dateobs_begin, # exposure date/time range start
     'dateobs_end=s'     => \$dateobs_end, # exposure date/time range stop
+    'camera=s'          => \$camera,     # Camera
     'dbname|d=s'        => \$dbname, # Database name
     'logfile=s'         => \$logfile,
-    'skip_burned'       => \$skip_burned,   # Print to stdout
-    'rerun_from_first'  => \$rerun_from_first,   # Print to stdout
     'verbose'           => \$verbose,   # Print to stdout
     'save-temps'        => \$save_temps, # Save temporary files?
+
     ) or pod2usage( 2 );
 
@@ -43 +43 @@
     defined $dbname;
 
-if ($skip_burned and $rerun_from_first) { &my_die("-rerun_from_first and -skip_burned are incompatible"); }
+unless (defined $camera) {
+    $camera = "GPC1";
+#    warn("No camera defined. Defaulting to $camera and warning.");
+}
 
 my $missing_tools;
@@ -49 +52 @@
 my $funpack  = can_run('funpack')  or (warn "Can't find funpack" and $missing_tools = 1);
 my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1);
-my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+#my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -57 +61 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+# IPP configuration (including nebulous)
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up");
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+# Determine the value of a "good" burntool run.
+my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run ( command => $config_cmd, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+}
+
+my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+    &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+
+my $burntoolStateGood = 999;
+foreach my $cfg (@$recipeData) {
+    if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
+        $burntoolStateGood = $cfg->{value};
+    }
+}
+if ($burntoolStateGood == 999) {
+    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+}
+my $outState = -1 * abs($burntoolStateGood);
+
+print ">>$burntoolStateGood<<\n";
+
+# Define list of images to examine.
 my $command = "$regtool -processedimfile";
 $command .= " -class_id $class_id";
@@ -64 +99 @@
 $command .= " -dbname $dbname" if defined $dbname;
 
-my @command = split /\s+/, $command;
-my ( $stdin, $stdout, $stderr ); # Buffers for running program
-print "Running [$command]...\n" if $verbose;
-if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
-    &my_die("Unable to perform regtool -processedimfile");
-}
-print $stdout . "\n" if $verbose;
-
-my @files;
-my @whole = split /\n/, $stdout;
-my @single = ();
-while ( scalar @whole > 0 ) {
-    my $value = shift @whole;
-    push @single, $value;
-    if ($value =~ /^\s*END\s*$/) {
-        push @single, "\n";
-
-        my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
-        &my_die("Unable to parse output from regtool") unless defined $list;
-        push @files, @$list;
-        @single = ();
-    }
-}
-
-# IPP configuration (including nebulous)
-my $ipprc = PS::IPP::Config->new() or my_die("Unable to set up");
-
-$ipprc->redirect_output($logfile) if $logfile;
-
-my $Nfiles = @files;
-print "files: $Nfiles\n";
+( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+    run ( command => $command, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
+}
+
+my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+    &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
+
+my $files = parse_md_list($metadata) or
+    &my_die("Unable to parse metadat list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
 
 my $REALRUN = 1;
-my $RAWTABLES = 1;
-# XXX if we want to avoid using the raw tables in this script, we need to delay deletion of the tempfile until the next pass.
-
-# first pass: mark the db entries to catch failures
-# these will not be identified as burned by chip processing (user_1 > 0.5)
-if (! $skip_burned)  {
-    foreach my $file (@files) {
+
+# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass
+my $processNext = 0;
+my $previousTable = '';
+
+foreach my $file (@$files) {
+    my $exp_id = $file->{exp_id};
+    my $state = $file->{burntool_state};
+
+    my $rawImfile = $file->{uri};
+    my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
+    
+    my $outTable  = $file->{uri};
+    $outTable  =~ s/fits$/burn.tbl/;
+    my $outTableReal = $ipprc->file_resolve($outTable, 1);
+    
+    my $process = 0;
+
+    if ($state == 0) {
+        $process = 1;
+    }
+    elsif ($state == -1) {
+        $process = 1;
+    }
+    elsif ($state == -3) {
+        $process = 1;
+    }
+    elsif ($state == -2) {
+        &my_die("Aborting as $rawImfile has modified pixel data!");
+    }
+    elsif ($state == $burntoolStateGood) {
+        $process = 0;
+        $previousTable = "infits=$rawImfileReal";
+    }
+    elsif ($state == -1 * $burntoolStateGood) {
+        $process = 0;
+        $previousTable = "in=$outTableReal";
+    }
+    else {
+        $process = 1;
+    }
+
+    if (($process == 1)||($processNext == 1)) {
+        $processNext = 1;
+        # Set state to processing
+        my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
+        if ($status) {
+            &my_die("failed to update imfile");
+        }
+        $file->{burntool_state} = -1;
         
-        # rerun_from_first treats the first image as already burned (if it is already burned)
-        # the artifact table from the first file is used for the rest of the sequence.
-        if ($rerun_from_first) {
-            $skip_burned = 1;
-            $rerun_from_first = 0;
-            next;
-        }
-        my $exp_id = $file->{exp_id};
-
-        my $status;
-        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 0.1", 1);
+        my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX",
+                                        DIR => '/tmp/',
+                                        UNLINK => !$save_temps,
+                                        SUFFIX => '.fits');
+        my $tempPixels = $tempfile->filename;
+        
+        # Run burntool
+        $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
+        if ($status) {
+            &my_die("failed on funpack",$exp_id,$class_id);
+        }
+        if ($previousTable ne '') {
+            $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN);
+        }
+        else {
+            $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
+        }
+        if ($status) {
+            &my_die("failed on burntool",$exp_id,$class_id);
+        }
+        
+        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
         if ($status) {
             &my_die("failed to update imfile");
         }
-        $file->{user_1} = 0.1;
-    }
-}
-
-my $prevFileOpt = "";
-foreach my $file (@files) {
-    my $exp_id = $file->{exp_id};
-
-    my $rawImfile = $file->{uri};
-    my $rawImfileReal;
-    if ($REALRUN) {
-        $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
-    } else {
-        ($rawImfileReal) = $rawImfile =~ m|^neb:/(.*)|;
-    }
-    print "rawImfile: $rawImfile -> $rawImfileReal\n";
-
-    # mangle name, create tmp file (always a UNIX file)
-    my $basename = `basename $rawImfile`; chomp $basename;
-    my $tempfile = new File::Temp ( TEMPLATE => "$basename.XXXX", 
-                                    DIR => '/tmp',
-                                    UNLINK => !$save_temps,
-                                    SUFFIX => '.fits');
-    my $tmpImfileReal = $tempfile->filename;
-    # print "tmpImfile: $tmpImfile -> $tmpImfileReal\n";
-    
-    # destination for the burntool-applied image file (may be a NEB file)
-    my $outImfile = $rawImfile;
-    $outImfile =~ s/fits$/burn.fits/;
-    my $outImfileReal = $ipprc->file_resolve($outImfile, 1);
-    # print "outImfile: $outImfile -> $outImfileReal\n";
-
-    # burntool now can write the artifacts to the fits file.
-    # destination for the burntool artifacts.  use this if RAWTABLES is set.
-    my $artImfile;
-    my $artImfileReal;
-    if ($RAWTABLES) {
-        $artImfile = $rawImfile;
-        $artImfile =~ s/fits$/burn.tbl/;
-        $artImfileReal = $ipprc->file_resolve($artImfile, 1);
-        # print "artImfile: $artImfile -> $artImfileReal\n";
-    }
-
-    print "$rawImfile : $skip_burned, $file->{user_1}\n";
-
-    if (! ($skip_burned and ($file->{user_1} > 0.5))) {
-        print "running on: $rawImfile\n";
-        # uncompress the image (do we need to check if it is compressed?)
-        my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN);
-        if ($status) {
-            &my_die("failed on funpack");
-        }
-
-        if ($RAWTABLES) {
-            $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt persist=t", $REALRUN);
-        } else {
-            $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt persist=t", $REALRUN);
-        }
-        if ($status) {
-            &my_die("failed on burntool");
-        }
-
-        # compress the image (do we need to check if it is compressed?)
-        $status = vsystem ("$fpack -S $tmpImfileReal > $outImfileReal", $REALRUN);
-        if ($status) {
-            &my_die("failed on fpack");
-        }
-
-        # set the user_1 value to a new result value for now
-        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 2.0", 1);
-        if ($status) {
-            &my_die("failed to update imfile");
-        }
-        print "\n";
-    }
-    # save the artifact file for the next image
-    if ($RAWTABLES) {
-        $prevFileOpt = "in=$artImfileReal";
-    } else {
-        $prevFileOpt = "infits=$artImfileReal";
-    }
-}
-
+
+        $previousTable = "in=$outTableReal";
+
+        print "\n";
+    }
+}
+    
 exit 0;
 
@@ -220 +214 @@
 sub my_die {
     my $message = shift;
-
+    if ($#_ != -1) {
+        my $exp_id = shift;
+        my $class_id = shift;
+        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
+    }
     printf STDERR "$message\n";
     exit 1;