Changeset 27840 for branches/simtest_nebulous_branches/tools/ipp_apply_burntool.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• tools/ipp_apply_burntool.pl (modified) (9 diffs, 1 prop)

branches/simtest_nebulous_branches/tools/ipp_apply_burntool.pl

@@ -1 +1 @@
 #!/usr/bin/env perl
 # this program is used to run 'burntool' on images for a single chip
-# for all exposures for a given time period.   
-# USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD 
+# for all exposures for a given time period.
+# USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD
 
 # XXX todo:
 # - add filters to processedimfile
-# 
+#
 
 use warnings;
@@ -12 +12 @@
 use Carp;
 
-use IPC::Cmd 0.36 qw( can_run );
-use IPC::Run 0.36 qw( run );
+use IPC::Cmd 0.36 qw( can_run run );
+#use IPC::Run 0.36 qw( run );
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
@@ -20 +20 @@
 use Pod::Usage qw( pod2usage );
 
-my ( $class_id, $dateobs_begin, $dateobs_end, $skip_burned, $rerun_from_first, $dbname, $logfile, $verbose, $save_temps);
+my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $camera, $dbname, $logfile, $verbose, $save_temps);
 GetOptions(
     'class_id=s'        => \$class_id, # chip identifier
     'dateobs_begin=s'   => \$dateobs_begin, # exposure date/time range start
     'dateobs_end=s'     => \$dateobs_end, # exposure date/time range stop
+    'camera=s'          => \$camera,     # Camera
     'dbname|d=s'        => \$dbname, # Database name
     'logfile=s'         => \$logfile,
-    'skip_burned'       => \$skip_burned,   # Print to stdout
-    'rerun_from_first'  => \$rerun_from_first,   # Print to stdout
     'verbose'           => \$verbose,   # Print to stdout
     'save-temps'        => \$save_temps, # Save temporary files?
+
     ) or pod2usage( 2 );
 
@@ -43 +43 @@
     defined $dbname;
 
-if ($skip_burned and $rerun_from_first) { &my_die("-rerun_from_first and -skip_burned are incompatible"); }
+unless (defined $camera) {
+    $camera = "GPC1";
+#    warn("No camera defined. Defaulting to $camera and warning.");
+}
 
 my $missing_tools;
@@ -49 +52 @@
 my $funpack  = can_run('funpack')  or (warn "Can't find funpack" and $missing_tools = 1);
 my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1);
-my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1);
+#my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -57 +62 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+# IPP configuration (including nebulous)
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up");
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+# Determine the value of a "good" burntool run.
+my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run ( command => $config_cmd, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+}
+
+my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+    &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+
+my $burntoolStateGood = 999;
+foreach my $cfg (@$recipeData) {
+    if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
+        $burntoolStateGood = $cfg->{value};
+    }
+}
+if ($burntoolStateGood == 999) {
+    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+}
+my $outState = -1 * abs($burntoolStateGood);
+
+print ">>$burntoolStateGood<<\n";
+
+# Define list of images to examine.
 my $command = "$regtool -processedimfile";
 $command .= " -class_id $class_id";
@@ -64 +100 @@
 $command .= " -dbname $dbname" if defined $dbname;
 
-my @command = split /\s+/, $command;
-my ( $stdin, $stdout, $stderr ); # Buffers for running program
-print "Running [$command]...\n" if $verbose;
-if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
-    &my_die("Unable to perform regtool -processedimfile");
-}
-print $stdout . "\n" if $verbose;
-
-my @files;
-my @whole = split /\n/, $stdout;
-my @single = ();
-while ( scalar @whole > 0 ) {
-    my $value = shift @whole;
-    push @single, $value;
-    if ($value =~ /^\s*END\s*$/) {
-        push @single, "\n";
-
-        my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
-        &my_die("Unable to parse output from regtool") unless defined $list;
-        push @files, @$list;
-        @single = ();
-    }
-}
-
-# IPP configuration (including nebulous)
-my $ipprc = PS::IPP::Config->new() or my_die("Unable to set up");
-
-$ipprc->redirect_output($logfile) if $logfile;
-
-my $Nfiles = @files;
-print "files: $Nfiles\n";
+( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run ( command => $command, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
+}
+
+my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+    &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
+
+my $files = parse_md_list($metadata) or
+    &my_die("Unable to parse metadata list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR);
 
 my $REALRUN = 1;
-my $RAWTABLES = 1;
-# XXX if we want to avoid using the raw tables in this script, we need to delay deletion of the tempfile until the next pass.
-
-# first pass: mark the db entries to catch failures
-# these will not be identified as burned by chip processing (user_1 > 0.5)
-if (! $skip_burned)  {
-    foreach my $file (@files) {
-        
-        # rerun_from_first treats the first image as already burned (if it is already burned)
-        # the artifact table from the first file is used for the rest of the sequence.
-        if ($rerun_from_first) {
-            $skip_burned = 1;
-            $rerun_from_first = 0;
-            next;
-        }
-        my $exp_id = $file->{exp_id};
-
-        my $status;
-        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 0.1", 1);
-        if ($status) {
-            &my_die("failed to update imfile");
-        }
-        $file->{user_1} = 0.1;
-    }
-}
-
-my $prevFileOpt = "";
-foreach my $file (@files) {
+
+# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass
+my $processNext = 0;
+my $previousTable = '';
+
+foreach my $file (@$files) {
     my $exp_id = $file->{exp_id};
+    my $state = $file->{burntool_state};
 
     my $rawImfile = $file->{uri};
-    my $rawImfileReal;
-    if ($REALRUN) {
-        $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
-    } else {
-        ($rawImfileReal) = $rawImfile =~ m|^neb:/(.*)|;
-    }
-    print "rawImfile: $rawImfile -> $rawImfileReal\n";
-
-    # mangle name, create tmp file (always a UNIX file)
-    my $basename = `basename $rawImfile`; chomp $basename;
-    my $tempfile = new File::Temp ( TEMPLATE => "$basename.XXXX", 
-                                    DIR => '/tmp',
-                                    UNLINK => !$save_temps);
-    my $tmpImfileReal = $tempfile->filename;
-    # print "tmpImfile: $tmpImfile -> $tmpImfileReal\n";
-    
-    # destination for the burntool-applied image file (may be a NEB file)
-    my $outImfile = $rawImfile;
-    $outImfile =~ s/fits$/burn.fits/;
-    my $outImfileReal = $ipprc->file_resolve($outImfile, 1);
-    # print "outImfile: $outImfile -> $outImfileReal\n";
-
-    # burntool now can write the artifacts to the fits file.
-    # destination for the burntool artifacts.  use this if RAWTABLES is set.
-    my $artImfile;
-    my $artImfileReal;
-    if ($RAWTABLES) {
-        $artImfile = $rawImfile;
-        $artImfile =~ s/fits$/burn.tbl/;
-        $artImfileReal = $ipprc->file_resolve($artImfile, 1);
-        # print "artImfile: $artImfile -> $artImfileReal\n";
-    }
-
-    print "$rawImfile : $skip_burned, $file->{user_1}\n";
-
-    if (! ($skip_burned and ($file->{user_1} > 0.5))) {
-        print "running on: $rawImfile\n";
-        # uncompress the image (do we need to check if it is compressed?)
-        my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN);
-        if ($status) {
-            &my_die("failed on funpack");
-        }
-
-        if ($RAWTABLES) {
-            $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN);
-        } else {
-            $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN);
-        }
-        if ($status) {
-            &my_die("failed on burntool");
-        }
-
-        # compress the image (do we need to check if it is compressed?)
-        $status = vsystem ("$fpack -S $tmpImfileReal > $outImfileReal", $REALRUN);
-        if ($status) {
-            &my_die("failed on fpack");
-        }
-
-        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 1.0", 1);
+    my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
+
+    my $outTable  = $file->{uri};
+    $outTable  =~ s/fits$/burn.tbl/;
+    my $outTableReal = $ipprc->file_resolve($outTable, 1);
+
+    my $process = 0;
+    my $previousTableStyle = 0;
+
+    if ($state == 0) {
+        $process = 1;
+    }
+    elsif ($state == -1) {
+        $process = 1;
+    }
+    elsif ($state == -3) {
+        $process = 1;
+    }
+    elsif ($state == -2) {
+        &my_die("Aborting as $rawImfile has modified pixel data!");
+    }
+    elsif ($state == $burntoolStateGood) {
+        $process = 0;
+        $previousTableStyle = 1;
+    }
+    elsif ($state == -1 * $burntoolStateGood) {
+        $process = 0;
+#       $previousTable = "in=$outTableReal";
+        $previousTableStyle = -1;
+    }
+    else {
+        $process = 1;
+    }
+    if ($REALRUN == 0) {
+        print "##Pretending to process as instructed!\n";
+        $process = 1;
+    }
+    if (($process == 1)||($processNext == 1)) {
+        $processNext = 1;
+        # Set state to processing
+        my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
         if ($status) {
             &my_die("failed to update imfile");
         }
+        $file->{burntool_state} = -1;
+
+        my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX",
+                                        DIR => '/tmp/',
+                                        UNLINK => !$save_temps,
+                                        SUFFIX => '.fits');
+        my $tempPixels = $tempfile->filename;
+
+        # Run burntool
+        $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
+        if ($status) {
+            &my_die("failed on funpack",$exp_id,$class_id);
+        }
+        if ($previousTable ne '') {
+            $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN);
+        }
+        else {
+            $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
+        }
+        if ($status) {
+            &my_die("failed on burntool",$exp_id,$class_id);
+        }
+        &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
+
+        $status = vsystem ("$nebXattr --write $outTable user.copies:2", $REALRUN);
+        if ($status) {
+            &my_die("failed to neb-xattr imfile", $outTable);
+        }
+        $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
+        if ($status) {
+            &my_die("failed to update imfile");
+        }
+
+        $previousTableStyle = -1;
+
         print "\n";
     }
-    # save the artifact file for the next image
-    if ($RAWTABLES) {
-        $prevFileOpt = "in=$artImfileReal";
-    } else {
-        $prevFileOpt = "infits=$artImfileReal";
-    }
+
+    if ($previousTableStyle == 1) {
+        $previousTable = "infits=$rawImfileReal";
+    }
+    elsif ($previousTableStyle == -1) {
+        $previousTable = "in=$outTableReal";
+    }
+    else {
+        die "previousTableStyle undefined!\n";
+    }
+
 }
 
@@ -207 +225 @@
 
     print "$command\n";
-    
+
     my $status = 0;
     if ($realrun) {
-        $status = system ($command);
+        $status = system ($command);
     }
 
@@ -218 +236 @@
 sub my_die {
     my $message = shift;
-
+    if ($#_ != -1) {
+        my $exp_id = shift;
+        my $class_id = shift;
+        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
+    }
     printf STDERR "$message\n";
     exit 1;