IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 17893


Ignore:
Timestamp:
Jun 3, 2008, 9:37:51 AM (18 years ago)
Author:
bills
Message:

Various changes to allow processing of detectability query requests
by the postage stamp server request processor

Location:
trunk/pstamp/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/pstamp_parser_run.pl

    r16975 r17893  
    55###
    66
     7#XXX TODO: accept dbname save-temps and verbose as command line parameters
     8#XXX see notes about error handling
     9
    710use warnings;
    811use strict;
    912
    1013use Sys::Hostname;
    11 my $host = hostname();
    12 print "\n\n";
    13 print "Starting script $0 on $host\n\n";
     14
     15my $verbose = 1;
     16if ($verbose) {
     17    my $host = hostname();
     18    print "\n\n";
     19    print "Starting script $0 on $host\n\n";
     20}
    1421
    1522if (@ARGV != 1) {
    … …  
    1825
    1926my $request_id = $ARGV[0];
    20 
    2127
    2228use IPC::Cmd 0.36 qw( can_run run );
    … …  
    5763my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
    5864my $pstampparse = can_run('pstampparse') or (warn "Can't find pstampparse" and $missing_tools = 1);
     65my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
    5966my $dsget = can_run('dsget') or (warn "Can't find dsget" and $missing_tools = 1);
    6067
    … …  
    7178    my $command = "$pstamptool -pendingreq -req_id $request_id";
    7279    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    73         run(command => $command, verbose => 1);
     80        run(command => $command, verbose => $verbose);
    7481    unless ($success) {
    7582        die("Unable to perform $command error code: $error_code");
    … …  
    8592    my $requests = parse_md_list($metadata);
    8693
    87     $psrequest = ${@$requests}[0];
     94    # TODO: We are assuming that we get an array of length 1 (or zero)
     95    # should we fail if we get more?
     96    #$psrequest = ${@$requests}[0];
     97    $psrequest = $requests->[0];
    8898}
    8999
    … …  
    112122        my $command = "$dsget --uri $uri --filename $new_uri";
    113123        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    114             run(command => $command, verbose => 1);
     124            run(command => $command, verbose => $verbose);
    115125        unless ($success) {
    116126            die("Unable to perform $command error code: $error_code");
    … …  
    121131}
    122132
    123 #  create the output diredtory
     133#  find the name of the output directory
    124134my $outdir;
    125135if ($ds_id) {
    126136    my $command = "$pstamptool -datastore -ds_id $ds_id";
    127137    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    128         run(command => $command, verbose => 1);
     138        run(command => $command, verbose => $verbose);
    129139    unless ($success) {
    130140        die("Unable to perform $command error code: $error_code");
    … …  
    140150    my $dataStores = parse_md_list($metadata);
    141151
    142     my $dataStore = ${@$dataStores}[0];
     152    my $dataStore = $dataStores->[0];
    143153
    144154    $outdir = "$outputDataStoreRoot/$psrequest->{outFileset}";
    … …  
    147157}
    148158
     159#  create the output directory
    149160if (! -e $outdir ) {
    150161    mkdir $outdir or die("unable to create output directory $outdir");
    … …  
    153164}
    154165
    155 # run pstampparse to parse the queue the jobs for this request
     166# run the appropriate parse command to parse the queue the jobs for this request
     167# first check the extension header to find the EXTNAME
     168my $request_type = find_request_type($uri);
     169
     170print STDERR "request_type is $request_type\n" if $verbose;
     171
     172my $parse_cmd;
     173if ($request_type eq "PS1_PS_REQUEST") {
     174    $parse_cmd = $pstampparse . " -mode queue_job -req_id $request_id -out_dir $outdir $uri";
     175} elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
     176    $parse_cmd = $dqueryparse  . " --mode queue_job --req_id $request_id --out_dir $outdir --uri $uri";
     177}
     178
     179if (!$parse_cmd) {
     180    # XXX TODO mark the error state for the job and set it to stop
     181    die "null request type found in $uri" if !$request_type;
     182    die "unknown request type $request_type found in $uri";
     183}
     184
    156185{
    157     my $command = "$pstampparse -mode queue_job -req_id $request_id -out_dir $outdir $uri";
    158     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    159         run(command => $command, verbose => 1);
    160     unless ($success) {
    161 #        die("Unable to perform $command error code: $error_code");
     186    my $command = "$parse_cmd";
     187    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     188        run(command => $command, verbose => $verbose);
     189    unless ($success) {
    162190        if (!open OUT, ">$outdir/parse_error") {
    163191            die("unable to open parse_error file $outdir/parse_error");
    … …  
    167195    }
    168196}
    169 
    170197# XXX make sure that a job got actually got queued. If not set the request result and set state to
    171198# run so that the "request finisher" can build the results file.
    … …  
    177204    my $command = "$pstamptool -processedreq -req_id $request_id -state run";
    178205    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    179         run(command => $command, verbose => 1);
     206        run(command => $command, verbose => $verbose);
    180207    unless ($success) {
    181208        die("Unable to perform $command error code: $error_code");
    … …  
    184211
    185212exit 0;
     213
     214sub find_request_type {
     215    # find the EXTNAME in the input fits table
     216    # TODO: do this right handling errors etc.
     217    my $file_name = shift;
     218    my $out = `echo $file_name | fields -x 0 EXTNAME`;
     219
     220    # output from fields is filename value
     221    my ($dummy, $extname) = split " ", $out;
     222
     223    return $extname;
     224}

  • trunk/pstamp/scripts/pstamp_queue_requests.pl

    r16592 r17893  
    55# the database of pending requests.
    66#
     7
     8# XXX TODO: need to take dbname as a paramter
    79
    810use warnings;
    … …  
    8991    my $outProduct = $ds->{outProduct};
    9092    my $ds_id = $ds->{ds_id};
    91     my @raw_output;
     93    my @lines;
    9294    {
    93         my $command = "$dsproductls --uri $ds->{uri}/index.txt --last_fileset $ds->{lastFileset}";
     95        my $command = "$dsproductls --uri $ds->{uri}/index.txt";
     96        $command .= " --last_fileset $ds->{lastFileset}" if $ds->{lastFileset};
    9497        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    9598            run(command => $command, verbose => $verbosity);
    … …  
    100103        if (@$stdout_buf == 0) {
    101104            print STDERR "no new request files in data store $ds_id\n";
    102             next;
     105            next; # next data store
    103106        }
    104         @raw_output = @$stdout_buf;
     107        my $out_buf = join("", @$stdout_buf);
     108        # split raw output into lines
     109        # XXX: why do I think that i need the map?
     110        #my @lines = map {split "\n"} $raw_output;
     111        @lines = split /^/, $out_buf;
    105112    }
    106113
    107     foreach my $fs (@raw_output) {
    108         # split into lines
    109         my @lines = map {split "\n"} $fs;
     114    #
     115    # each line contains a fileset
     116    #
     117    foreach my $line (@lines) {
     118        # parse the line into fields split by whitespace
     119        my ($uri, $fs_name, $date, $type) = split " ", $line;
     120
     121        next if ( $uri =~ /^#.*/);
     122
     123        my @files;
     124        {
     125            my $command = "$dsfilesetls --uri $uri";
     126            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     127                    run(command => $command, verbose => $verbosity);
     128            unless ($success) {
     129                die("Unable to perform $command: $error_code");
     130            }
     131
     132            if (@$stdout_buf == 0) {
     133                print STDERR "no file sets found\n";
     134                next;
     135            }
     136            my $out_buf = join "", @$stdout_buf;
     137            @files = split /^/, $out_buf;
     138        }
    110139
    111140        #
    112         # each line contains a fileset
     141        # For each file in the fileset add a request
    113142        #
    114         foreach my $line (@lines) {
    115             # parse the line into fields split by whitespace
    116             my ($uri, $fs_name, $date, $type) = split " ", $line;
    117             $_ = $uri;
     143        foreach my $file (@files) {
     144            my ($req_uri, $fn, $size, $md5sum, $type, $chipname) = split " ", $file;
     145
    118146            # skip comment lines
    119             if (/^#.*/) {
    120                 next;
     147            next if $req_uri =~ (/^#.*/);
     148            {
     149                my $command = "$pstamptool -addreq -uri $req_uri -ds_id $ds_id -out_fileset $outProduct/$fs_name";
     150                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     151                        run(command => $command, verbose => $verbosity);
     152                    unless ($success) {
     153                            # XXX: what do we do now?
     154                            die("Unable to perform $command: $error_code");
     155                    }
    121156            }
    122 
    123             my @out_buf;
    124             {
    125                 my $command = "$dsfilesetls --uri $uri";
    126                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    127                     run(command => $command, verbose => $verbosity);
    128                 unless ($success) {
    129                     die("Unable to perform $command: $error_code");
    130                 }
    131 
    132                 if (@$stdout_buf == 0) {
    133                     print STDERR "no file sets found\n";
    134                     next;
    135                 }
    136                 @out_buf = @$stdout_buf;
    137             }
    138 
    139             #
    140             # This is getting indented pretty far
    141             # Looks like we're going to need to learn about functions/subroutines :)
    142             #
    143 
    144             ## TODO: fail if there is more than one request file in the fileset
    145             foreach my $bit (@out_buf) {
    146                 my @files = map {split "\n"} $bit;
    147                 #
    148                 # For each file in the fileset add a request
    149                 #
    150                 foreach my $file (@files) {
    151                     my ($req_uri, $fn, $size, $md5sum, $type, $chipname) = split " ", $file;
    152 
    153                     # skip comment lines
    154                     $_ = $req_uri;
    155                     if (/^#.*/) {
    156                         next;
    157                     }
    158 
    159                     {
    160                         my $command = "$pstamptool -addreq -uri $req_uri -ds_id $ds_id -out_fileset $outProduct/$fs_name";
    161                         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    162                             run(command => $command, verbose => $verbosity);
    163                         unless ($success) {
    164                                 # XXX: what do we do now?
    165                                 die("Unable to perform $command: $error_code");
    166                         }
    167                     }
    168                     $lastFileset = $fs_name;
    169                     # we only allow one request file per fileset
    170                     last;
    171                 }
    172             }
     157            $lastFileset = $fs_name;
     158            # we only allow one request file per fileset
     159            # XXX TODO: maybe Relax this restriction
     160            last;
    173161        }
    174162    }
Note: See TracChangeset for help on using the changeset viewer.