Changeset 30317 for trunk/pstamp/scripts/pstamp_job_run.pl

Timestamp:

Jan 20, 2011, 10:28:02 AM (15 years ago)

Author:

bills

Message:

Implement extraction of postage stamps from uncensored images either by using the backup images
or by replacing the censored pixels from destreak recovery images.

File:

• trunk/pstamp/scripts/pstamp_job_run.pl (modified) (8 diffs)

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -14 +14 @@
 use File::Basename;
 use File::Copy;
+use File::Temp qw(tempfile);
 use Digest::MD5::File qw( file_md5_hex );
 use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
 use POSIX;
@@ -26 +28 @@
 
 my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
-my ($verbose, $dbname, $dbserver, $no_update);
+my ($verbose, $dbname, $dbserver, $no_update, $save_temps);
 
 GetOptions(
@@ -39 +41 @@
     'verbose'           =>  \$verbose,
     'no-update'         =>  \$no_update,
+    'save-temps'        =>  \$save_temps,
 );
 
@@ -75 +78 @@
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
 my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
+my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
+my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -88 +93 @@
     my $argString;
     $argString = $params->{job_args};
+    my $stage = $params->{stage};
 
     # XXX: should we do any other sanity checking?
     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
 
-    $argString .= " -file $params->{image}";
+    my $nan_masked = 1;
+    my $muggle = 0;
+    if (!$params->{magicked}) {
+        $nan_masked = 0;
+    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
+        # Attempt to find or create a muggle image
+        $nan_masked = 0;
+        $muggle = 1;
+    }
+
+    my $image = $params->{image};
+    my $mask;
+    my $variance;
+    my $fileArgs = " -file $params->{image}";
     my @file_list = ($params->{image});
     
-    my $nan_masked = 1;
-    if (!$params->{magicked}) {
-        $nan_masked = 0;
-    }
     if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
-        $argString .= " -mask $params->{mask}";
-        push @file_list, $params->{mask};
+        $mask = $params->{mask};
+        $fileArgs .= " -mask $mask";
+        push @file_list, $mask;
     }
     if ($options & $PSTAMP_SELECT_VARIANCE) {
-        $argString .= " -variance $params->{weight}";
-        push @file_list, $params->{weight};
+        $variance = $params->{weight};
+        $fileArgs .= " -variance $variance";
+        push @file_list, $variance;
     }
 
@@ -113 +130 @@
     }
 
-    check_files(@file_list);
-
-    my $command = "$ppstamp $outputBase $argString";
+    # check that actual input files exist
+    check_files($PSTAMP_GONE, @file_list);
+
+    # find our output directory
+    my $outdir = dirname($outputBase);
+    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
+    
+    if ($muggle) {
+        # first see if the original uncensored images are around
+        if (check_for_backups($params, \$fileArgs)) {
+            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
+            print "Making stamps from backup images\n";
+        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
+            # user required uncensored but since stage isn't chip we can't rebuild them
+            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE);
+        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) {
+            # make stamps from uncensored images
+            $muggle = 0;
+        } else {
+            # Try and replace the streaks from the recovery images
+
+            @file_list = ();
+            $tmproot = "$outdir/$job_id";
+            mkdir $tmproot or
+                my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
+            # find the "directory" of the input path_base
+            my $inputdir = dirname($image);
+            my $base = basename($image);
+            $tmpImage = "$tmproot/$base";
+
+            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
+            # stage work. Import the rule here.
+            my $recImage = "$inputdir/REC_$base";
+            my $newFileArgs = " -file $tmpImage";
+            $muggle_command .= " -image $image -recimage $recImage";
+            push @file_list, $recImage;
+
+            if ($mask) {
+                $base = basename($mask);
+                $tmpMask = "$tmproot/$base";
+                $newFileArgs .= " -mask $tmpMask";
+                my $recMask = "$inputdir/REC_$base";
+                $muggle_command .= " -mask $mask -recmask $recMask";
+                push @file_list, $recMask;
+            }
+
+            if ($variance) {
+                $base = basename($variance);
+                $tmpVariance = "$tmproot/$base";
+                $newFileArgs .= " -weight $tmpVariance";
+                my $recVariance = "$inputdir/REC_$base";
+                $muggle_command .= " -weight $variance -recweight $recVariance";
+                push @file_list, $recVariance;
+            }
+            if (check_files(0, @file_list)) {
+                # recovery files exist and are accessible restore the excised pixels
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $muggle_command, verbose => $verbose);
+                unless ($success) {
+                    my $exitStatus = WEXITSTATUS($error_code);
+                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
+                }
+
+                # set the ppstamp file arguments
+                $fileArgs = $newFileArgs;
+            } else {
+                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
+                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
+                }
+                # just make stamps from the censored images
+                print "Unable to restore uncensored images, will extract stamps from censored images\n";
+                # these files won't be used so zap them
+                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
+            }
+        }
+    }
+
+    my $command = "$ppstamp $outputBase $argString $fileArgs";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
-    $command .= " -stage $params->{stage}" if $params->{stage};
+    $command .= " -stage $stage";
     $command .= " -no_censor_masked" unless $nan_masked;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -131 +225 @@
     }
 
+    if (!$save_temps) {
+        unlink $tmpImage if $tmpImage;
+        unlink $tmpMask if $tmpMask;
+        unlink $tmpVariance if $tmpVariance;
+        rmdir $tmproot if $tmproot;
+    }
+
     if ($exitStatus == 0) {
-        my $dir = dirname($outputBase);
-
-        my $reglist = "$dir/reglist$job_id";
+        my $reglist = "$outdir/reglist$job_id";
 
         my $F;
@@ -363 +462 @@
 
 sub check_files {
+    my $error_code = shift;
+    my $return_code = 1;
     foreach my $f (@_) {
         if (!$ipprc->file_exists($f)) {
-            my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop');
-        }
-    }
+            if ($error_code) {
+                my_die( "file $f does not exist:", $job_id, $error_code, 'stop');
+            } else {
+                print STDERR "file $f does not exist\n";
+                $return_code = 0;
+            }
+        }
+    }
+    return $return_code;
+}
+
+sub check_for_backups {
+    my $params = shift;
+    my $r_fileArgs = shift;
+
+    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
+    my $results = runToolAndParse($command, $verbose);
+    my $dsComponent  = $results->[0];
+
+    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
+
+        print "magicDSFile state is full. Backup images should exist\n";
+
+        # replace the file names with the backup paths
+        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
+        my @args = split " ", $$r_fileArgs;
+
+        my $newFileArgs;
+        while (@args) {
+            my $a = shift @args;
+            my $f = shift @args;
+            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
+
+            my $backup_image_name = $ipprc->destreaked_filename($f);
+            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
+            $newFileArgs .= " $a $backup_image_name";
+        }
+        $$r_fileArgs = $newFileArgs;
+
+        return 1;
+    }
+
+    return 0;
 }