IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 7, 2009, 12:45:24 PM (17 years ago)
Author:
eugene
Message:

update from mainline

Location:
branches/eam_branches/20090715
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/20090715

  • branches/eam_branches/20090715/pstamp/scripts/pstamp_job_run.pl

    r23267 r25022  
    1414use Digest::MD5::File qw( file_md5_hex );
    1515use PS::IPP::PStamp::RequestFile qw( :standard );
    16 
    17 my $verbose;
    18 my $dbname;
    19 my $job_id;
    20 my $redirect_output;
    21 my $output_base;
     16use IPC::Cmd 0.36 qw( can_run run );
     17
     18use PS::IPP::Metadata::Config;
     19#use PS::IPP::Metadata::Stats;
     20#use PS::IPP::Metadata::List qw( parse_md_list );
     21
     22use PS::IPP::Config qw( :standard );
     23
     24my ($job_id, $redirect_output, $outputBase, $rownum, $jobType);
     25my ($verbose, $dbname, $dbserver, $no_update);
     26
     27GetOptions(
     28    'job_id=s'          =>  \$job_id,
     29    'job_type=s'        =>  \$jobType,
     30    'rownum=s'          =>  \$rownum,
     31    'output_base=s'     =>  \$outputBase,
     32    'redirect-output'   =>  \$redirect_output,
     33    'dbname=s'          =>  \$dbname,
     34    'dbserver=s'        =>  \$dbserver,
     35    'verbose'           =>  \$verbose,
     36    'no-update'         =>  \$no_update,
     37);
     38
    2239
    2340my $host = hostname();
    24 
    25 
    26 GetOptions(
    27     'job_id=s'      =>  \$job_id,
    28     'output_base=s' =>  \$output_base,
    29     'redirect-output' => \$redirect_output,
    30     'dbname=s'      =>  \$dbname,
    31     'verbose'       =>  \$verbose,
    32 );
    33 
    34 
    3541if ($verbose) {
    3642    print "\n\n";
    … …  
    3945
    4046die "job_id is required" if !$job_id;
    41 die "output_base is required" if !$output_base;
    42 
    43 use IPC::Cmd 0.36 qw( can_run run );
    44 
    45 use PS::IPP::Metadata::Config;
    46 use PS::IPP::Metadata::Stats;
    47 use PS::IPP::Metadata::List qw( parse_md_list );
    48 
    49 use PS::IPP::Config qw($PS_EXIT_SUCCESS
    50                        $PS_EXIT_UNKNOWN_ERROR
    51                        $PS_EXIT_SYS_ERROR
    52                        $PS_EXIT_CONFIG_ERROR
    53                        $PS_EXIT_PROG_ERROR
    54                        $PS_EXIT_DATA_ERROR
    55                        $PS_EXIT_TIMEOUT_ERROR
    56                        metadataLookupStr
    57                        metadataLookupBool
    58                        caturi
    59                        );
     47
     48my_die( "job_type is required", $job_id, $PS_EXIT_PROG_ERROR) if !$jobType;
     49my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum;
     50my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
     51
     52
    6053my $ipprc = PS::IPP::Config->new(); # IPP Configuration
    61 
    6254if ($redirect_output) {
    63     my $logDest = "$output_base.log";
    64     $ipprc->redirect_output($logDest);
     55    my $logDest = "$outputBase.log";
     56    $ipprc->redirect_output($logDest)
     57        or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     58}
     59
     60
     61if (!$dbserver) {
     62    $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
    6563}
    6664
    … …  
    7270
    7371if ($missing_tools) {
    74     warn("Can't find required tools.");
    75     exit ($PS_EXIT_CONFIG_ERROR);
     72    my_die("Can't find required tools", $job_id, $PS_EXIT_CONFIG_ERROR);
    7673}
    7774
    7875my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    7976
    80 my $psjob;
    81 #Look up the uri for the given job
    82 {
    83     my $command = "$pstamptool -pendingjob -job_id $job_id";
     77my $jobStatus;
     78if ($jobType eq "stamp") {
     79    my $argslist = "$outputBase.args";
     80    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     81    my $argString = <ARGSLIST>;
     82    close ARGSLIST;
     83    chomp $argString;
     84
     85    # XXX: should we do any other sanity checking?
     86    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
     87
     88    my $command = "$ppstamp $outputBase $argString";
    8489    $command .= " -dbname $dbname" if $dbname;
     90    $command .= " -dbserver $dbserver" if $dbserver;
    8591    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    8692        run(command => $command, verbose => $verbose);
    87     unless ($success) {
    88         die("Unable to perform pstamptool -pendingjob: $error_code");
    89     }
    90 
    91     if (@$stdout_buf == 0) {
    92         print STDERR "pending pstamp job id $job_id not found\n";
    93         exit 0;
    94     }
    95     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    96         die("Unable to parse metdata config doc");
    97 
    98     my $jobs = parse_md_list($metadata);
    99     my $num = @$jobs;
    100     die "unexpected number of jobs $num found for job: $job_id" if $num > 1;
    101 
    102     $psjob = $jobs->[0];
    103 }
    104 
    105 if (!$psjob) {
    106     print STDERR "postage stamp job $job_id not found\n";
    107     exit 1;
    108 }
    109 
    110 my $rownum = $psjob->{rownum};
    111 my $uri = $psjob->{uri};
    112 my $outputBase = $psjob->{outputBase};
    113 my $argString = $psjob->{args};
    114 my $jobType = $psjob->{jobType};
    115 
    116 my $jobStatus;
    117 if ($jobType eq "stamp") {
    118     my $command = "$ppstamp -file $uri $outputBase $argString";
    119     $command .= " -dbname $dbname" if $dbname;
    120     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    121         run(command => $command, verbose => $verbose);
    12293
    12394    if ($success) {
    124         # XXX shouldn't need to do this, review schema`
    12595        my $dir = dirname($outputBase);
    12696
    12797        my $reglist = "$dir/reglist$job_id";
    12898
    129         open F, ">$reglist" or die "can't open $reglist for output";
    130 
    131         # figure out what output images we should expect
    132 
    133         # Note: we are assuming the contents of the filerules here.
    134         my @extensions = ( "fits", "mk.fits", "wt.fits");
    135 
    136         # we always require an image
     99        open F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     100
     101        # Figure out what output images were produced
     102
     103        # Note: we are assuming the contents of the PSTAMP filerules here.
     104        my %extensions = ( $PSTAMP_SELECT_IMAGE  => "fits",
     105                           $PSTAMP_SELECT_MASK   => "mk.fits",
     106                           $PSTAMP_SELECT_WEIGHT => "wt.fits");
     107
     108        # we always create a stamp of the image
    137109        my $output_mask = $PSTAMP_SELECT_IMAGE;
    138110
    139         # we search the argString for -mask and -weight.
     111        # we search the argString for -mask and -variance.
    140112        # searching the arg string allows us to avoid adding a column in pstampJob
    141         # XXX yeah but does this make us susceptible to user input fouling us up?
    142         # For example, what if somebody sets the output base to file-mask-weight
    143         # TODO: clean this up
    144         $output_mask |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
    145         $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/);
    146 
    147         # XXX we're getting a bit intimate with the bit field definitions here.
    148         # do better use a hash
    149 
    150         my $m = 1;
    151         foreach my $extension (@extensions) {
    152             my $do_this_one = $m & $output_mask;
    153             $m = $m << 1;
     113        $output_mask   |= $PSTAMP_SELECT_MASK   if ($argString =~ /-mask/);
     114        $output_mask   |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-variance/);
     115
     116        foreach my $key (keys (%extensions)) {
     117            my $do_this_one = $key & $output_mask;
     118
    154119            next if (! $do_this_one);
     120
     121            my $extension = $extensions{$key};
    155122
    156123            my $basename = basename($outputBase);
    … …  
    167134    } else {
    168135        $jobStatus = $error_code >> 8;
    169         print STDERR "ppstamp failed with error code: $jobStatus\n";
     136        my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus);
    170137    }
    171138} elsif ($jobType eq "get_image") {
     139    my_die( "get_image jobs not working right now", $job_id, $PS_EXIT_CONFIG_ERROR);
     140
     141    my $uri = "";
    172142    my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum";
    173143    $command .= " --dbname $dbname" if $dbname;
     144    $command .= " --dbserver $dbserver" if $dbserver;
    174145    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    175146        run(command => $command, verbose => $verbose);
    … …  
    179150    } else {
    180151        $jobStatus = $error_code >> 8;
    181         print STDERR "ppstamp failed with error code: $jobStatus\n";
     152        my_die( "pstamp_get_image_job failed with error code: $jobStatus", $job_id, $jobStatus);
    182153    }
    183154} elsif ($jobType eq "detect_query") {
    184     die("multiple detect_query jobs not supported yet");
     155    my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR);
    185156} else {
    186     die("unknown jobType $jobType found");
    187 }
    188 
    189 # stop the job and set the result value
     157    my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
     158}
     159
     160# mark the job stopped in the database
    190161{
    191     my $command = "$pstamptool -updatejob -job_id $job_id -state stop -fault $jobStatus";
     162    my $command = "$pstamptool -updatejob -job_id $job_id -state stop";
    192163    $command .= " -dbname $dbname" if $dbname;
    193     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    194         run(command => $command, verbose => $verbose);
    195     unless ($success) {
    196         die("Unable to perform $command: $error_code");
    197     }
    198 }
    199 
    200 exit $jobStatus;
     164    $command .= " -dbserver $dbserver" if $dbserver;
     165    if (!$no_update) {
     166        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     167            run(command => $command, verbose => $verbose);
     168        unless ($success) {
     169            die("Unable to perform $command: $error_code");
     170        }
     171    } else {
     172        print STDERR "skipping command: $command\n"
     173    }
     174}
     175
     176exit 0;
    201177
    202178# create a string to be passed as input to dsreg when registering this file in a fileset
    … …  
    209185    if (-e $path) {
    210186        my @finfo = stat($path);
    211         die "failed to stat $path" unless (@finfo);    # XXX clean up
     187        my_die("failed to stat $path", $job_id, $PS_EXIT_UNKNOWN_ERROR) unless (@finfo);
    212188        my $bytes = $finfo[7];
    213189        my $md5sum = file_md5_hex($path);
    … …  
    215191        return "$filename|$bytes|$md5sum|$filetype|";
    216192    } else {
    217         die "$filename not found at $path";
    218     }
    219 }
     193        my_die("$filename not found at $path", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     194    }
     195}
     196
     197sub my_die
     198{
     199    my $msg = shift;            # Warning message on die
     200    my $job_id = shift;         # job identifier
     201    my $exit_code = shift;      # Exit code to add
     202
     203    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     204
     205    carp($msg);
     206    if (defined $job_id and not $no_update) {
     207        my $command = "$pstamptool -updatejob";
     208        $command .= " -job_id $job_id";
     209        $command .= " -fault $exit_code";
     210        $command .= " -dbname $dbname" if defined $dbname;
     211        $command .= " -dbserver $dbserver" if defined $dbserver;
     212        system($command);
     213    }
     214    exit $exit_code;
     215}
     216
Note: See TracChangeset for help on using the changeset viewer.