Index: branches/czw_branch/20101203/pstamp/scripts/Makefile.am =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/Makefile.am (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/Makefile.am (revision 30586) @@ -17,7 +17,9 @@ pstamp_runcommand.sh \ pstamp_server_status \ + pstamp_save_server_status.pl \ pstamp_webrequest.pl \ pstamp_get_image_job.pl \ psmkreq \ + psstatus \ pstampstopfaulted \ pstamp_checkdependent.pl \ Index: branches/czw_branch/20101203/pstamp/scripts/psstatus =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/psstatus (revision 30586) +++ branches/czw_branch/20101203/pstamp/scripts/psstatus (revision 30586) @@ -0,0 +1,283 @@ +#!/bin/env perl + +use strict; +use warnings; + +use DBI; +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use File::Temp qw(tempfile); + +my $limit = 0; +my $running; +my $finished; +my $req_faulted; +my $job_faulted; +my $dbname; +my $dbserver; +my $dbuser; +my $dbpassword; +my $verbose; +my $finishing; + +GetOptions( + 'running|r', \$running, + 'finished|f', \$finished, + 'finishing', \$finishing, + 'req_faulted', \$req_faulted, + 'job_faulted', \$job_faulted, + 'limit|l=i', \$limit, + 'dbname=s', \$dbname, + 'verbose|v', \$verbose, +) or pod2usage (2); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV ; + +$running = 1 if (!$finished and !$finishing and !$running); + +my $no_args = ! (defined $running or defined $finished or defined $req_faulted or defined $job_faulted ); + +die "cannot supply --running and --finished\n" if ($running and $finished); + + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +my $running_query = " +SELECT + unfinishedReq.req_id, + name, + label, + reqType, + priority, + state, + fault, + rowcount AS 'Num Rows', + jobcount AS 'Total Jobs', + IFNULL(completedJobs,0) AS 'Completed Jobs', + IFNULL(runningJobs, 0) AS 'Pending Jobs', + IFNULL(faultedJobs, 0) AS 'Faulted Jobs', + IFNULL(depcount, 0) AS 'Image updates' + , timestamp AS 'last state change (UTC)' +FROM ( + SELECT + req_id, + name, + label, + reqType, + pstampRequest.state, + pstampRequest.fault, + timestamp, + priority +FROM pstampRequest +JOIN Label USING(label) + WHERE pstampRequest.state = 'run' or pstampRequest.state ='new' + -- OR pstampRequest.state = 'run.wait' +) as unfinishedReq +LEFT JOIN ( + SELECT req_id, + count(job_id) AS jobcount, + count(distinct rownum) AS rowcount +FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' + OR pstampRequest.state = 'run.wait') + GROUP BY req_id +) as rowCounts +ON unfinishedReq.req_id = rowCounts.req_id + +LEFT JOIN ( + SELECT req_id, + count(distinct dep_id) AS depcount +FROM pstampRequest + JOIN pstampJob USING(req_id) + JOIN pstampDependent USING(dep_id) + WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' + OR pstampRequest.state = 'run.wait') + AND dep_id > 0 + AND pstampDependent.state = 'full' + GROUP BY req_id +) as depCounts +ON unfinishedReq.req_id = depCounts.req_id + +LEFT JOIN ( + SELECT req_id, + count(job_id) AS runningJobs + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE (pstampRequest.state = 'run' or pstampRequest.state ='new' + OR pstampRequest.state = 'run.wait') + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 + GROUP BY req_id +) as runningJobs +ON unfinishedReq.req_id = runningJobs.req_id + +LEFT JOIN ( + SELECT req_id, + count(job_id) AS faultedJobs + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE (pstampRequest.state = 'run' or pstampRequest.state ='new' + OR pstampRequest.state = 'run.wait') + -- AND pstampJob.state = 'run' + AND pstampJob.fault > 0 + GROUP BY req_id +) as faultedJobs +ON unfinishedReq.req_id = faultedJobs.req_id +LEFT JOIN ( + SELECT req_id, + count(job_id) AS completedJobs + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE (pstampRequest.state = 'run' or pstampRequest.state ='new' + OR pstampRequest.state = 'run.wait') + AND pstampJob.state = 'stop' + GROUP BY req_id +) as finishedJobs +ON unfinishedReq.req_id = finishedJobs.req_id + +ORDER by unfinishedReq.req_id +"; + +my $finishing_query = " +SELECT + req_id, + name, + label, + reqType, + pstampRequest.state, + pstampRequest.fault, + count(job_id) as 'Finished jobs', + timestamp as 'last state change' +FROM pstampRequest + JOIN pstampJob USING(req_id) +WHERE pstampRequest.state ='run' + AND ( + SELECT count(job_id) FROM pstampJob + WHERE pstampJob.req_id = pstampRequest.req_id + AND pstampJob.state != 'stop' + ) = 0 +GROUP BY req_id +ORDER by req_id +"; + +my $finished_query = " +SELECT + finishedReq.req_id, + name, + label, + reqType, + rowcount AS 'Num Rows', + jobcount AS 'Total Jobs', + IFNULL(success, 0) AS 'Successful Jobs', + IFNULL(faulted,0) AS 'Faulted Jobs', + IFNULL(depcount, 0) AS 'Image updates completed', + timestamp AS 'Completion Time (UTC)' +FROM ( + SELECT + req_id, + name, + label, + reqType, +-- pstampRequest.state, +-- pstampRequest.fault, + timestamp +FROM pstampRequest + WHERE pstampRequest.state = 'stop' + AND date_add(timestamp, interval 1 day) >=utc_timestamp() +) as finishedReq + +LEFT JOIN ( + SELECT req_id, + count(job_id) AS jobcount, + count(distinct rownum) AS rowcount +FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state = 'stop' + AND date_add(timestamp, interval 1 day) >=utc_timestamp() + GROUP BY req_id +) as rowCounts +ON finishedReq.req_id = rowCounts.req_id + +LEFT JOIN ( + SELECT req_id, + count(distinct dep_id) AS depcount +FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state = 'stop' + AND date_add(timestamp, interval 1 day) >=utc_timestamp() + AND dep_id > 0 + GROUP BY req_id +) as depCounts +ON finishedReq.req_id = depCounts.req_id + +LEFT JOIN ( + SELECT req_id, + count(job_id) AS success + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state = 'stop' + AND date_add(timestamp, interval 1 day) >=utc_timestamp() + AND pstampJob.fault = 0 + GROUP BY req_id +) as successfulJobs +ON finishedReq.req_id = successfulJobs.req_id +LEFT JOIN ( + SELECT req_id, + count(job_id) AS faulted + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state = 'stop' + AND date_add(timestamp, interval 1 day) >=utc_timestamp() + AND pstampJob.fault > 0 + GROUP BY req_id +) as faultedJobs +ON finishedReq.req_id = faultedJobs.req_id +ORDER by timestamp desc +"; + + +my $sql; +if ($finished) { + $sql = $finished_query; +} elsif ($running) { + $sql = $running_query; +} elsif ($finishing) { + $sql = $finishing_query; +} else { + die "can't happen"; +} + +if ($limit and !$no_args) { + $sql .= " LIMIT $limit"; +} + +{ + print STDERR "$sql\n" if $verbose; + my @args = ( "--table", "--user=$dbuser", "--host=$dbserver", "--password=$dbpassword", $dbname); + + push @args, '--verbose' if $verbose; + + open my $mysql_pipe, "|-", 'mysql', @args or die "failed to run mysql\n"; + print $mysql_pipe "$sql;\n"; + close $mysql_pipe; + +} + +exit 0; + + +sub getDBHandle { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "PS_DBSERVER"); + $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBUSER"); + $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBPASSWORD"); + if (!$dbname) { + $dbname = metadataLookupStr($ipprc->{_siteConfig}, "PS_DBNAME"); + } + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + + Index: branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30586) @@ -690,4 +690,6 @@ if ($dsfile->{data_state} eq 'cleaned') { $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c"; + # XXX: get the recoveryroot from a config file (it isn't actually used except to check whether it is in nebulous) + $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover"; $command .= " -set_label $rlabel" if $rlabel; if (!$no_update) { Index: branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl (revision 30586) @@ -14,6 +14,8 @@ use File::Basename; use File::Copy; +use File::Temp qw(tempfile); use Digest::MD5::File qw( file_md5_hex ); use PS::IPP::PStamp::RequestFile qw( :standard ); +use PS::IPP::PStamp::Job qw( :standard ); use IPC::Cmd 0.36 qw( can_run run ); use POSIX; @@ -26,5 +28,5 @@ my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); -my ($verbose, $dbname, $dbserver, $no_update); +my ($verbose, $dbname, $dbserver, $no_update, $save_temps); GetOptions( @@ -39,4 +41,5 @@ 'verbose' => \$verbose, 'no-update' => \$no_update, + 'save-temps' => \$save_temps, ); @@ -75,4 +78,6 @@ my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1); +my $streaksreplace = can_run('streaksreplace') or (warn "Can't find streaksreplace" and $missing_tools = 1); +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); if ($missing_tools) { @@ -88,22 +93,34 @@ my $argString; $argString = $params->{job_args}; + my $stage = $params->{stage}; # XXX: should we do any other sanity checking? my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; - $argString .= " -file $params->{image}"; + my $nan_masked = 1; + my $muggle = 0; + if (!$params->{magicked}) { + $nan_masked = 0; + } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) { + # Attempt to find or create a muggle image + $nan_masked = 0; + $muggle = 1; + } + + my $image = $params->{image}; + my $mask; + my $variance; + my $fileArgs = " -file $params->{image}"; my @file_list = ($params->{image}); - my $nan_masked = 1; - if (!$params->{magicked}) { - $nan_masked = 0; - } if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) { - $argString .= " -mask $params->{mask}"; - push @file_list, $params->{mask}; + $mask = $params->{mask}; + $fileArgs .= " -mask $mask"; + push @file_list, $mask; } if ($options & $PSTAMP_SELECT_VARIANCE) { - $argString .= " -variance $params->{weight}"; - push @file_list, $params->{weight}; + $variance = $params->{weight}; + $fileArgs .= " -variance $variance"; + push @file_list, $variance; } @@ -113,10 +130,87 @@ } - check_files(@file_list); - - my $command = "$ppstamp $outputBase $argString"; + # check that actual input files exist + check_files($PSTAMP_GONE, @file_list); + + # find our output directory + my $outdir = dirname($outputBase); + my ($tmpImage, $tmpMask, $tmpVariance, $tmproot); + + if ($muggle) { + # first see if the original uncensored images are around + if (check_for_backups($params, \$fileArgs)) { + # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images + print "Making stamps from backup images\n"; + } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) { + # user required uncensored but since stage isn't chip we can't rebuild them + my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE); + } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) { + # make stamps from uncensored images + $muggle = 0; + } else { + # Try and replace the streaks from the recovery images + + @file_list = (); + $tmproot = "$outdir/$job_id"; + mkdir $tmproot or + my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR); + my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot"; + # find the "directory" of the input path_base + my $inputdir = dirname($image); + my $base = basename($image); + $tmpImage = "$tmproot/$base"; + + # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and + # stage work. Import the rule here. + my $recImage = "$inputdir/REC_$base"; + my $newFileArgs = " -file $tmpImage"; + $muggle_command .= " -image $image -recimage $recImage"; + push @file_list, $recImage; + + if ($mask) { + $base = basename($mask); + $tmpMask = "$tmproot/$base"; + $newFileArgs .= " -mask $tmpMask"; + my $recMask = "$inputdir/REC_$base"; + $muggle_command .= " -mask $mask -recmask $recMask"; + push @file_list, $recMask; + } + + if ($variance) { + $base = basename($variance); + $tmpVariance = "$tmproot/$base"; + $newFileArgs .= " -weight $tmpVariance"; + my $recVariance = "$inputdir/REC_$base"; + $muggle_command .= " -weight $variance -recweight $recVariance"; + push @file_list, $recVariance; + } + if (check_files(0, @file_list)) { + # recovery files exist and are accessible restore the excised pixels + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $muggle_command, verbose => $verbose); + unless ($success) { + my $exitStatus = WEXITSTATUS($error_code); + my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus); + } + + # set the ppstamp file arguments + $fileArgs = $newFileArgs; + } else { + if ($options & $PSTAMP_REQUIRE_UNCENSORED) { + my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE); + } + # just make stamps from the censored images + print "Unable to restore uncensored images, will extract stamps from censored images\n"; + # these files won't be used so zap them + ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef); + } + } + } + + my $command = "$ppstamp $outputBase $argString $fileArgs"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; - $command .= " -stage $params->{stage}" if $params->{stage}; + $command .= " -stage $stage"; $command .= " -no_censor_masked" unless $nan_masked; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -131,8 +225,13 @@ } + if (!$save_temps) { + unlink $tmpImage if $tmpImage; + unlink $tmpMask if $tmpMask; + unlink $tmpVariance if $tmpVariance; + rmdir $tmproot if $tmproot; + } + if ($exitStatus == 0) { - my $dir = dirname($outputBase); - - my $reglist = "$dir/reglist$job_id"; + my $reglist = "$outdir/reglist$job_id"; my $F; @@ -363,9 +462,51 @@ sub check_files { + my $error_code = shift; + my $return_code = 1; foreach my $f (@_) { if (!$ipprc->file_exists($f)) { - my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop'); - } - } + if ($error_code) { + my_die( "file $f does not exist:", $job_id, $error_code, 'stop'); + } else { + print STDERR "file $f does not exist\n"; + $return_code = 0; + } + } + } + return $return_code; +} + +sub check_for_backups { + my $params = shift; + my $r_fileArgs = shift; + + my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}"; + my $results = runToolAndParse($command, $verbose); + my $dsComponent = $results->[0]; + + if ($dsComponent and ($dsComponent->{data_state} eq 'full')) { + + print "magicDSFile state is full. Backup images should exist\n"; + + # replace the file names with the backup paths + # fileArgs has the form: -file imagename [-mask maskname] [-weight weightname] + my @args = split " ", $$r_fileArgs; + + my $newFileArgs; + while (@args) { + my $a = shift @args; + my $f = shift @args; + my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f; + + my $backup_image_name = $ipprc->destreaked_filename($f); + my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name; + $newFileArgs .= " $a $backup_image_name"; + } + $$r_fileArgs = $newFileArgs; + + return 1; + } + + return 0; } Index: branches/czw_branch/20101203/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstamp_parser_run.pl (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/pstamp_parser_run.pl (revision 30586) @@ -31,4 +31,5 @@ my $outdir; my $label; +my $need_magic; my $verbose; my $dbname; @@ -41,4 +42,5 @@ 'outdir=s' => \$outdir, 'label=s' => \$label, + 'need_magic' => \$need_magic, 'redirect-output' => \$redirect_output, 'verbose' => \$verbose, @@ -160,4 +162,5 @@ $parse_cmd = "$pstampparse"; $parse_cmd .= " --label $label" if $label; + $parse_cmd .= " --need_magic" if $need_magic; $request_fault = 0; } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") { Index: branches/czw_branch/20101203/pstamp/scripts/pstamp_save_server_status.pl =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstamp_save_server_status.pl (revision 30586) +++ branches/czw_branch/20101203/pstamp/scripts/pstamp_save_server_status.pl (revision 30586) @@ -0,0 +1,74 @@ +#!/bin/env perl +# +# run pstamp_server_status and save the results in a date stamped file +# optionally change the symlink in the web directory to point to the new file + +use strict; +use warnings; + +use PS::IPP::Config 1.01 qw( :standard ); +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# XXX: We should get this from site.config, but that would about double +# the execution time for this script. +my $rundir = "/data/ippc17.0/pstamp"; + +my $status_file = "$rundir/web/status.html"; + +my $updatelink; +my $verbose; +my $save_temps; + +GetOptions( + 'rundir=s' => \$rundir, + 'update-link' => \$updatelink, + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +my $missing_tools; +my $pstamp_server_status = can_run('pstamp_server_status') + or (warn "Can't find pstamp_server_status" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($sec, $min, $hour, $mday, $month, $year) = gmtime; +$year += 1900; +$month += 1; + +my $dir = sprintf "$rundir/work/server_status/%4d/%02d/%02d", $year, $month, $mday; + +if (!-e $dir ) { + my $rc = system "mkdir -p $dir"; + if ($rc) { + my $status = $rc >> 8; + print STDERR "mkdir failed error: $status\n"; + exit $status; + } +} + +my $file = sprintf "$dir/pstamp.status.%4d-%02d-%02dT%02d:%02d:%02d.html", + $year, $month, $mday, $hour, $min, $sec; + +my $command = "pstamp_server_status > $file"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or 1); + warn("$command failed. exit status: $error_code"); + exit $error_code; +} + +if ($updatelink) { + if (-e $status_file) { + unlink $status_file or die "failed to unlink existing status file $status_file\n"; + } + + symlink $file, $status_file or die "failed to update staus file link $status_file"; +} + +exit 0; Index: branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status (revision 30586) @@ -25,8 +25,4 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -#pod2usage( -msg => "Required options: --first --stage", -# -exitval => 3) unless -# defined $first and -# defined $stage; # Look for programs we need @@ -39,9 +35,24 @@ } +# XXX: Note under pantasks this will be the curent directory, so this +# isn't necessary if the pstamp/web stuff is configured to point at the +# ipp build. But currently it's running out of Bill's build my $ipphome = "/home/panstarrs/ipp"; +my $status_cmd = "psstatus"; + $rundir = "$ipphome/pstamp" if !$rundir; chdir $rundir or die "failed to cd to $rundir"; + +my $down = 0; +if ($down) { + print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n"; + exit 0; +} + +my $now = `date -u`; + +print "Status updated:     $now

\n"; my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps); @@ -132,13 +143,24 @@ print "Task pstamp.job.run not found.
\n"; } + + # now run the script psstatus to check the status of running requests and finished requests print "
Unfinished Requests
\n"; print "
\n";
-    system "/home/panstarrs/bills/ipp/tools/psstatus -r ";
+    system "$status_cmd --running";
     print "
\n"; + +if (0) { + # these are included in running requests above + print "
Requests building results fileset
\n"; + print "
\n";
+    system "$status_cmd --finishing";
+    print "
\n"; +} + print "
Requests completed in last 24 hours (most recently completed first)
\n"; print "
\n";
-    # finished requests
-    system "/home/panstarrs/bills/ipp/tools/psstatus -f";
+    system "$status_cmd --finished";
     print "
\n"; + } else { print "Task pstamp.request.run not found.\n"; @@ -147,4 +169,3 @@ - exit 0; Index: branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl =================================================================== --- branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl (revision 30118) +++ branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl (revision 30586) @@ -30,4 +30,5 @@ my $save_temps; my $no_update; +my $dest_requires_magic; GetOptions( @@ -38,4 +39,5 @@ 'label=s' => \$label, 'mode=s' => \$mode, + 'need_magic'=> \$dest_requires_magic, 'dbname=s' => \$dbname, 'dbserver=s'=> \$dbserver, @@ -126,5 +128,6 @@ } -# Adjust the label for requests coming in over the web interaface + +# Adjust the label for requests coming in over the web interface my $label_changed = 0; @@ -384,6 +387,6 @@ my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); if (!$proj_hash) { - foreach $row (@$rowList) { - insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT); + foreach my $r (@$rowList) { + insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT); $num_jobs++; } @@ -409,15 +412,38 @@ $need_magic = $proj_hash->{need_magic}; - # Temporary hack so that IFA can get at non-magicked data - my $allow_mops_unmagicked = 1; - if ($allow_mops_unmagicked) { - if ($product and (($product eq "mops-pstamp-results") or - ($product eq "mops-pstamp-results2") or - ($product eq "ifa-pstamp-results"))) { - $need_magic = 0; - } - } - $need_magic = 0 if $stage eq 'stack'; + + if ($need_magic) { + + # this project requires that postage stamps be extracted from destreaked images + + if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) { + # The user has requested uncensored stamps + + if (!$dest_requires_magic) { + # and this user's data store destination is allowed uncensored stamps, so accept the request + $need_magic = 0; + } else { + print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n"; + if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) { + # user required uncensored stamps. Can't do it so fail. + foreach my $r (@$rowList) { + insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED); + $num_jobs++; + } + return $num_jobs; + } + + # user will accept censored stamps. alter OPTION_MASK and continue + + print STDERR " Will attempt to make destreaked stamps\n"; + # zap the offending bit in the option mask + $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED); + foreach my $r (@$rowList) { + $r->{OPTION_MASK} = $option_mask; + } + } + } + } my $numRows = scalar @$rowList;