Index: /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 30316) +++ /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 30317) @@ -17,4 +17,5 @@ resolve_project getCamRunByCamID + runToolAndParse parse_md_fast ); Index: /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30316) +++ /trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30317) @@ -29,4 +29,6 @@ $PSTAMP_SELECT_INVERSE $PSTAMP_SELECT_UNCONV + $PSTAMP_REQUEST_UNCENSORED + $PSTAMP_REQUIRE_UNCENSORED $PSTAMP_USE_IMFILE_ID $PSTAMP_NO_WAIT_FOR_UPDATE @@ -44,4 +46,5 @@ $PSTAMP_NO_JOBS_QUEUED $PSTAMP_NO_OVERLAP + $PSTAMP_NOT_AUTHORIZED ); our %EXPORT_TAGS = (standard => [@EXPORT_OK]); @@ -64,5 +67,7 @@ our $PSTAMP_USE_IMFILE_ID = 16384; -our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; +our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768; +our $PSTAMP_REQUEST_UNCENSORED = 1 << 16; +our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17; # job and result codes @@ -82,4 +87,5 @@ our $PSTAMP_NO_JOBS_QUEUED = 27; our $PSTAMP_NO_OVERLAP = 28; +our $PSTAMP_NOT_AUTHORIZED = 29; @@ -119,4 +125,5 @@ PSTAMP_NO_JOBS_QUEUED PSTAMP_NO_OVERLAP +PSTAMP_NOT_AUTHORIZED ); Index: /trunk/ippTasks/pstamp.pro =================================================================== --- /trunk/ippTasks/pstamp.pro (revision 30316) +++ /trunk/ippTasks/pstamp.pro (revision 30317) @@ -258,4 +258,5 @@ book getword pstampRequest $pageName ds_outProduct -var PRODUCT book getword pstampRequest $pageName outdir -var OUTDIR + book getword pstampRequest $pageName need_magic -var NEED_MAGIC book getword pstampRequest $pageName label -var LABEL @@ -263,4 +264,8 @@ $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output + + if ($NEED_MAGIC != 0) + $run = $run --need_magic + end add_standard_args run Index: /trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 30316) +++ /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 30317) @@ -680,4 +680,6 @@ foreach my $c (@$components) { my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c"; + # XXX: get this from a config file + $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover"; my $dsfile = runToolAndParseExpectOne($command, $verbose); if (!$dsfile) { Index: /trunk/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_job_run.pl (revision 30316) +++ /trunk/pstamp/scripts/pstamp_job_run.pl (revision 30317) @@ -14,6 +14,8 @@ use File::Basename; use File::Copy; +use File::Temp qw(tempfile); use Digest::MD5::File qw( file_md5_hex ); use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); use IPC::Cmd 0.36 qw( can_run run ); use POSIX; @@ -26,5 +28,5 @@ my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); -my ($verbose, $dbname, $dbserver, $no_update); +my ($verbose, $dbname, $dbserver, $no_update, $save_temps); GetOptions( @@ -39,4 +41,5 @@ 'verbose' => \$verbose, 'no-update' => \$no_update, + 'save-temps' => \$save_temps, ); @@ -75,4 +78,6 @@ my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1); +my $streaksreplace = can_run('streaksreplace') or (warn "Can't find streaksreplace" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); if ($missing_tools) { @@ -88,22 +93,34 @@ my $argString; $argString = $params->{job_args}; + my $stage = $params->{stage}; # XXX: should we do any other sanity checking? my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; - $argString .= " -file $params->{image}"; + my $nan_masked = 1; + my $muggle = 0; + if (!$params->{magicked}) { + $nan_masked = 0; + } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) { + # Attempt to find or create a muggle image + $nan_masked = 0; + $muggle = 1; + } + + my $image = $params->{image}; + my $mask; + my $variance; + my $fileArgs = " -file $params->{image}"; my @file_list = ($params->{image}); - my $nan_masked = 1; - if (!$params->{magicked}) { - $nan_masked = 0; - } if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) { - $argString .= " -mask $params->{mask}"; - push @file_list, $params->{mask}; + $mask = $params->{mask}; + $fileArgs .= " -mask $mask"; + push @file_list, $mask; } if ($options & $PSTAMP_SELECT_VARIANCE) { - $argString .= " -variance $params->{weight}"; - push @file_list, $params->{weight}; + $variance = $params->{weight}; + $fileArgs .= " -variance $variance"; + push @file_list, $variance; } @@ -113,10 +130,87 @@ } - check_files(@file_list); - - my $command = "$ppstamp $outputBase $argString"; + # check that actual input files exist + check_files($PSTAMP_GONE, @file_list); + + # find our output directory + my $outdir = dirname($outputBase); + my ($tmpImage, $tmpMask, $tmpVariance, $tmproot); + + if ($muggle) { + # first see if the original uncensored images are around + if (check_for_backups($params, \$fileArgs)) { + # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images + print "Making stamps from backup images\n"; + } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) { + # user required uncensored but since stage isn't chip we can't rebuild them + my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE); + } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) { + # make stamps from uncensored images + $muggle = 0; + } else { + # Try and replace the streaks from the recovery images + + @file_list = (); + $tmproot = "$outdir/$job_id"; + mkdir $tmproot or + my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR); + my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot"; + # find the "directory" of the input path_base + my $inputdir = dirname($image); + my $base = basename($image); + $tmpImage = "$tmproot/$base"; + + # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and + # stage work. Import the rule here. + my $recImage = "$inputdir/REC_$base"; + my $newFileArgs = " -file $tmpImage"; + $muggle_command .= " -image $image -recimage $recImage"; + push @file_list, $recImage; + + if ($mask) { + $base = basename($mask); + $tmpMask = "$tmproot/$base"; + $newFileArgs .= " -mask $tmpMask"; + my $recMask = "$inputdir/REC_$base"; + $muggle_command .= " -mask $mask -recmask $recMask"; + push @file_list, $recMask; + } + + if ($variance) { + $base = basename($variance); + $tmpVariance = "$tmproot/$base"; + $newFileArgs .= " -weight $tmpVariance"; + my $recVariance = "$inputdir/REC_$base"; + $muggle_command .= " -weight $variance -recweight $recVariance"; + push @file_list, $recVariance; + } + if (check_files(0, @file_list)) { + # recovery files exist and are accessible restore the excised pixels + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $muggle_command, verbose => $verbose); + unless ($success) { + my $exitStatus = WEXITSTATUS($error_code); + my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus); + } + + # set the ppstamp file arguments + $fileArgs = $newFileArgs; + } else { + if ($options & $PSTAMP_REQUIRE_UNCENSORED) { + my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE); + } + # just make stamps from the censored images + print "Unable to restore uncensored images, will extract stamps from censored images\n"; + # these files won't be used so zap them + ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef); + } + } + } + + my $command = "$ppstamp $outputBase $argString $fileArgs"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; - $command .= " -stage $params->{stage}" if $params->{stage}; + $command .= " -stage $stage"; $command .= " -no_censor_masked" unless $nan_masked; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -131,8 +225,13 @@ } + if (!$save_temps) { + unlink $tmpImage if $tmpImage; + unlink $tmpMask if $tmpMask; + unlink $tmpVariance if $tmpVariance; + rmdir $tmproot if $tmproot; + } + if ($exitStatus == 0) { - my $dir = dirname($outputBase); - - my $reglist = "$dir/reglist$job_id"; + my $reglist = "$outdir/reglist$job_id"; my $F; @@ -363,9 +462,51 @@ sub check_files { + my $error_code = shift; + my $return_code = 1; foreach my $f (@_) { if (!$ipprc->file_exists($f)) { - my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop'); - } - } + if ($error_code) { + my_die( "file $f does not exist:", $job_id, $error_code, 'stop'); + } else { + print STDERR "file $f does not exist\n"; + $return_code = 0; + } + } + } + return $return_code; +} + +sub check_for_backups { + my $params = shift; + my $r_fileArgs = shift; + + my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}"; + my $results = runToolAndParse($command, $verbose); + my $dsComponent = $results->[0]; + + if ($dsComponent and ($dsComponent->{data_state} eq 'full')) { + + print "magicDSFile state is full. Backup images should exist\n"; + + # replace the file names with the backup paths + # fileArgs has the form: -file imagename [-mask maskname] [-weight weightname] + my @args = split " ", $$r_fileArgs; + + my $newFileArgs; + while (@args) { + my $a = shift @args; + my $f = shift @args; + my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f; + + my $backup_image_name = $ipprc->destreaked_filename($f); + my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name; + $newFileArgs .= " $a $backup_image_name"; + } + $$r_fileArgs = $newFileArgs; + + return 1; + } + + return 0; } Index: /trunk/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 30316) +++ /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 30317) @@ -31,4 +31,5 @@ my $outdir; my $label; +my $need_magic; my $verbose; my $dbname; @@ -41,4 +42,5 @@ 'outdir=s' => \$outdir, 'label=s' => \$label, + 'need_magic' => \$need_magic, 'redirect-output' => \$redirect_output, 'verbose' => \$verbose, @@ -160,4 +162,5 @@ $parse_cmd = "$pstampparse"; $parse_cmd .= " --label $label" if $label; + $parse_cmd .= " --need_magic" if $need_magic; $request_fault = 0; } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { Index: /trunk/pstamp/scripts/pstamp_server_status =================================================================== --- /trunk/pstamp/scripts/pstamp_server_status (revision 30316) +++ /trunk/pstamp/scripts/pstamp_server_status (revision 30317) @@ -44,4 +44,10 @@ chdir $rundir or die "failed to cd to $rundir"; + +my $down = 0; +if ($down) { + print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n"; + exit 0; +} my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps); Index: /trunk/pstamp/scripts/pstampparse.pl =================================================================== --- /trunk/pstamp/scripts/pstampparse.pl (revision 30316) +++ /trunk/pstamp/scripts/pstampparse.pl (revision 30317) @@ -30,4 +30,5 @@ my $save_temps; my $no_update; +my $dest_requires_magic; GetOptions( @@ -38,4 +39,5 @@ 'label=s' => \$label, 'mode=s' => \$mode, + 'need_magic'=> \$dest_requires_magic, 'dbname=s' => \$dbname, 'dbserver=s'=> \$dbserver, @@ -126,5 +128,6 @@ } -# Adjust the label for requests coming in over the web interaface + +# Adjust the label for requests coming in over the web interface my $label_changed = 0; @@ -384,6 +387,6 @@ my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); if (!$proj_hash) { - foreach $row (@$rowList) { - insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + foreach my $r (@$rowList) { + insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT); $num_jobs++; } @@ -409,15 +412,38 @@ $need_magic = $proj_hash->{need_magic}; - # Temporary hack so that IFA can get at non-magicked data - my $allow_mops_unmagicked = 1; - if ($allow_mops_unmagicked) { - if ($product and (($product eq "mops-pstamp-results") or - ($product eq "mops-pstamp-results2") or - ($product eq "ifa-pstamp-results"))) { - $need_magic = 0; - } - } - $need_magic = 0 if $stage eq 'stack'; + + if ($need_magic) { + + # this project requires that postage stamps be extracted from destreaked images + + if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) { + # The user has requested uncensored stamps + + if (!$dest_requires_magic) { + # and this user's data store destination is allowed uncensored stamps, so accept the request + $need_magic = 0; + } else { + print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n"; + if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) { + # user required uncensored stamps. Can't do it so fail. + foreach my $r (@$rowList) { + insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED); + $num_jobs++; + } + return $num_jobs; + } + + # user will accept censored stamps. alter OPTION_MASK and continue + + print STDERR " Will attempt to make destreaked stamps\n"; + # zap the offending bit in the option mask + $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED); + foreach my $r (@$rowList) { + $r->{OPTION_MASK} = $option_mask; + } + } + } + } my $numRows = scalar @$rowList;