Changeset 35412 for branches/eam_branches/ipp-20130307/pstamp/scripts

Timestamp:

Apr 19, 2013, 3:39:39 PM (13 years ago)

Author:

eugene

Message:

merged from trunk

Location:

branches/eam_branches/ipp-20130307/pstamp/scripts

Files:

• . (modified) (1 prop)
• psmkreq (modified) (13 diffs)
• psstatus (modified) (6 diffs)
• pstamp_cleanup.pl (modified) (1 diff)
• pstamp_finish.pl (modified) (4 diffs)
• pstamp_job_run.pl (modified) (4 diffs)
• pstamp_parser_run.pl (modified) (3 diffs)
• pstamp_request_file (modified) (4 diffs)
• pstamp_webrequest.pl (modified) (6 diffs)
• pstampparse.pl (modified) (34 diffs)

branches/eam_branches/ipp-20130307/pstamp/scripts/psmkreq

@@ -28 +28 @@
 my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
 
-my ($image, $mask, $variance, $cmf, $psf, $backmdl, $inverse);
-my ($unconvolved, $use_imfile_id, $no_wait);
+my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse);
+my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait);
+
+# new header keywords for version 2
+my $action = 'PROCESS';
+my $email = 'null';
 
 my $default_size = 100;
@@ -52 +56 @@
 my $comment;
 
+# new request specification columns for version 2
+my $survey_name ='null';
+my $release_name = 'null';
+my $run_type ='null';
+my $fwhm_min = 0;
+my $fwhm_max = 0;
+
+
 my $missing_tools;
 my $pstamp_request_file  = can_run('pstamp_request_file')  or (warn "Can't find required program pstamp_request_file"  and $missing_tools = 1);
@@ -62 +74 @@
 GetOptions(
     'list=s'            => \$list,          # list of coordinates if undef ra and dec or x and y are required
-    'ra=s'              => \$ra,            # 
+    'ra=s'              => \$ra,
     'dec=s'             => \$dec,
     'x=s'               => \$x,
@@ -87 +99 @@
     'comment=s'         => \$comment,
 
+    'survey=s'          => \$survey_name,
+    'release=s'         => \$release_name,
+    'run_type=s'        => \$run_type,
+    'fwhm_min=s'        => \$fwhm_min,
+    'fwhm_max=s'        => \$fwhm_max,
+
     'option_mask=i'     => \$option_mask,
     'image'             => \$image,
     'mask'              => \$mask,
+    'jpeg'              => \$jpeg,
     'variance'          => \$variance,
     'cmf'               => \$cmf,
@@ -96 +115 @@
     'inverse'           => \$inverse,
     'unconvolved'       => \$unconvolved,
+    'convolved'         => \$convolved,
+    'uncompressed'      => \$uncompressed,
     'use_imfile_id'     => \$use_imfile_id,
     'do_not_wait'       => \$no_wait,
+
+    'action=s'          => \$action,
+    'email=s'           => \$email,
 
     'verbose'           => \$verbose,
@@ -156 +180 @@
 }
 
+if ($stage eq 'stack') {
+    # default is to do convolved stack
+    unless (defined $option_mask || $convolved) {
+        $unconvolved = 1;
+    }
+}
+
 checkFilter($filter, 'null', $filter)  if $filter;
 checkMJD($mjd_min, 0, "") if $mjd_min;
@@ -167 +198 @@
         $option_mask |= $PSTAMP_SELECT_MASK     if $mask;
         $option_mask |= $PSTAMP_SELECT_VARIANCE if $variance;
+        $option_mask |= $PSTAMP_SELECT_JPEG     if $jpeg;
+
+        # if no image was requested make a stamp of the image
         $option_mask = $PSTAMP_SELECT_IMAGE    if $option_mask == 0;
 
-        # if no image was requested make a stamp of the image
 
         $option_mask |= $PSTAMP_SELECT_CMF      if $cmf;
@@ -207 +240 @@
 } else {
     $rows = [];
-    push @$rows, buildRow("", $comment, $x, $y, $filter, $mjd_min, $mjd_max);
+    push @$rows, buildRow("", $comment, 1, $x, $y, $filter, $mjd_min, $mjd_max);
 }
 
 my ($tdf, $table_def_name) = tempfile ("/tmp/tabledef.XXXX", UNLINK => !$save_temps);
-print $tdf "$req_name 1\n";
+my $line = "$req_name 2 $action $email";
+print "$line\n" if $verbose;
+print $tdf "$line\n";
 my $rownum = 0;
 foreach my $row (@$rows) {
     $rownum++;
     my $line = "$rownum $row->{ra}\t$row->{dec}\t$width $height"
-        . " $coord_mask $job_type $option_mask $project $req_type"
+        . " $coord_mask $job_type $option_mask $project $survey_name $release_name $req_type"
         . " $stage $id $tess_id $component $data_group"
-        . " $row->{filter} $row->{mjd_min} $row->{mjd_max}";
+        . " $row->{filter} $row->{mjd_min} $row->{mjd_max} $run_type $fwhm_min $fwhm_max";
 
     if ($row->{comment} and $row->{comment} ne '') {
@@ -235 +270 @@
     $command .= " --output $output" if $output;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+        run(command => $command, verbose => $verbose);
     unless ($success) {
         print STDERR @$stderr_buf;
@@ -274 +309 @@
 
     my $row = {};
-    $row->{ra}      = checkRA($vals[0], $linenumber);
-    $row->{dec}     = checkDEC($vals[1], $linenumber);
-    $row->{filter}  = checkFilter($vals[2], $filter, $linenumber);
-    $row->{mjd_min} = checkMJD($vals[3], $mjd_min, $linenumber);;
-    $row->{mjd_max} = checkMJD($vals[4], $mjd_max, $linenumber);;
+    $row->{rownum}  = checkRownum($vals[0], $linenumber);
+    $row->{ra}      = checkRA($vals[1], $linenumber);
+    $row->{dec}     = checkDEC($vals[2], $linenumber);
+    $row->{filter}  = checkFilter($vals[3], $filter, $linenumber);
+    $row->{mjd_min} = checkMJD($vals[4], $mjd_min, $linenumber);;
+    $row->{mjd_max} = checkMJD($vals[5], $mjd_max, $linenumber);;
     $row->{comment} = $comment;
 
@@ -305 +341 @@
     my $linenumber = shift;
 
-    my $result;
+   my $result;
     if ($c =~ /\:/) {
         # sexagesmial format not valid for pixel coordinates
@@ -329 +365 @@
 }
 
+sub checkRownum {
+    my $rownum = shift;
+    my $linenumber = shift;
+    die "rownum can not be null at $linenumber\n" unless defined $rownum;
+
+    # XXX: For now just check that rownum is defined and a single word
+    # extend this. Should we require a number?
+    my @words = split " ", $rownum;
+    die "$rownum is not a vaild rownum at $linenumber\n" unless (scalar @words) == 1;
+}
+
 sub checkRA {
     my $ra = shift;
@@ -335 +382 @@
 
     if ($ra =~ /\:/) {
+        # assume RA is in hours:minutes:seconds
         return $checked * 360. / 24.;
     } else {

branches/eam_branches/ipp-20130307/pstamp/scripts/psstatus

@@ -68 +68 @@
     IFNULL(depcount, 0) AS 'Image updates'
     , timestamp AS 'last state change (UTC)'
+    , registered AS 'Submit time (UTC)'
 FROM (
     SELECT
@@ -77 +78 @@
         pstampRequest.fault,
         timestamp,
+        registered,
         IFNULL(priority,10000) AS priority
 FROM pstampRequest
 LEFT JOIN Label USING(label)
     WHERE pstampRequest.state = 'run' or pstampRequest.state ='new' 
+        OR pstampRequest.state = 'parsed'
    --     OR pstampRequest.state = 'run.wait'
 ) as unfinishedReq
@@ -89 +92 @@
 FROM pstampRequest JOIN pstampJob USING(req_id)
     WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' 
-        OR pstampRequest.state = 'run.wait')
+        OR pstampRequest.state = 'run.wait'
+        OR pstampRequest.state = 'parsed')
     GROUP BY req_id
 ) as rowCounts
@@ -101 +105 @@
     JOIN pstampDependent USING(dep_id) 
     WHERE (pstampRequest.state = 'run' or pstampRequest.state = 'new' 
+            OR pstampRequest.state = 'parsed'
             OR pstampRequest.state = 'run.wait')
         AND dep_id > 0
@@ -178 +183 @@
     IFNULL(faulted,0) AS 'Faulted Jobs',
     IFNULL(depcount, 0) AS 'Image updates completed',
-    timestamp AS 'Completion Time (UTC)'
+    timestamp AS 'Completion Time (UTC)',
+    registered AS 'Submit Time (UTC)'
 FROM (
     SELECT
@@ -187 +193 @@
 --    pstampRequest.state,
 --    pstampRequest.fault,
-    timestamp
+    timestamp,
+    registered
 FROM pstampRequest
     WHERE pstampRequest.state = 'stop'

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_cleanup.pl

@@ -183 +183 @@
     }
 
-    my $command = "rm -r $dir";
+    # do an ls of the directory before starting
+#    print "directory listing for $dir\n";
+#    system "ls $dir";
+
+    my $command = "rm -rf $dir";
     unless ($no_update) {
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         unless ($success) {
-            my $rc = $error_code >> 8;
-            my_die("Unable to perform $command return status: $rc", $req_id, $rc);
+            if (-e $dir) {
+                my $rc = $error_code >> 8;
+                my_die("Unable to perform $command return status: $rc", $req_id, $rc);
+            } else {
+                print STDERR "rm claimed to fail but directory no longer exists\n";
+            }
         }
     } else {

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_finish.pl

@@ -107 +107 @@
     my ($header, $rows) = read_request_file($req_file);
 
-    my $action = $header->{ACTION};
-
-    if (!$header or (($action eq 'PROCESS') and !$rows)) {
+    if (!$header or !$rows) {
         # Since a request got queued, the request file must have been readable at some point 
         my_die("failed to read request file $req_file", $req_id, $PS_EXIT_CONFIG_ERROR);
@@ -128 +126 @@
 
     my $request_fault = 0;
-    if ($action eq 'PROCESS') {
+    {
         # The results table definition file
         my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
@@ -238 +236 @@
                 } else {
                     my_die("No reglist for successful job: $job_id", $req_id, $PS_EXIT_PROG_ERROR) 
-                        if $fault eq $PSTAMP_SUCCESS;
+                        if $job->{state} eq 'stop' and $fault eq $PSTAMP_SUCCESS;
                     print STDERR "no reglist file for job $job_id\n" if $verbose;
                     print $tdf "$rownum|$fault|$error_string|0|$job_id|";
@@ -273 +271 @@
             }
         }
-    } elsif ($action ne 'LIST') {
-        my_die("Unexpected action $action found", $req_id, $PS_EXIT_PROG_ERROR);
-    } else {
-        # pstampparse did all of the work
     }
 

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_job_run.pl

@@ -113 +113 @@
 
     if ($stage eq "raw") {
+        # zap options that don't apply to raw stage
         $options &= ~($PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE); 
     }
@@ -136 +137 @@
         $argString .= " -astrom $params->{astrom}";
         push @file_list, $params->{astrom};
+    }
+
+    if ($options & $PSTAMP_SELECT_SOURCES) {
+        # Extract sources from astrometry file if provided. This will be the smf for chip stage
+        # or the skycal cmf for stacks
+        if ($params->{astrom}) {
+            $argString .= " -write_cmf";
+            if ($stage eq 'stack') {
+                # Set psphot recipe to STACKPHOT so that the extended source paramters will
+                # be copied from the cmf file.
+                $argString .= " -recipe PSPHOT STACKPHOT"
+            }
+        } elsif ($params->{cmf}) {
+            $argString .= " -write_cmf";
+            push @file_list, $params->{cmf};
+        } else {
+            print "Could not find suitable sources file will not write cmf\n";
+        }
     }
 
@@ -254 +273 @@
                            $PSTAMP_SELECT_MASK     => "mk.fits",
                            $PSTAMP_SELECT_VARIANCE => "wt.fits",
+                           $PSTAMP_SELECT_SOURCES  => "cmf",
                            $PSTAMP_SELECT_JPEG     => "jpg");
 
-        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG);
+        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
 
         foreach my $key (keys (%extensions)) {
@@ -400 +420 @@
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $cmf_file;
-        if ($stage ne 'chip') {
-            # we don't ship chip stage cmf files because they may not be censored
-            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
-        }
+#        if ($stage ne 'chip') {
+#            # we don't ship chip stage cmf files because they may not be censored
+#            $cmf_file = $params->{cmf} if ($options & $PSTAMP_SELECT_CMF);
+#        }
 
         my $outdir = dirname($output_base);

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_parser_run.pl

@@ -158 +158 @@
 my $request_fault = $PSTAMP_INVALID_REQUEST;
 
+# default action is to process the request after parsing. This can be overridden by
+# PREVIEW mode for pstamp requests
+my $action = 'PROCESS';
+
 if (-r $uri) {
     # run the appropriate parse command to parse the queue the jobs for this request
     # first check the extension header to find the EXTNAME
-    $request_type = find_request_type($uri);
+    $request_type = find_request_type($uri, \$action);
 
     if ($request_type) {
@@ -240 +244 @@
 
     if ($success) {
-        $newState = 'run';
+        # XXX: This bit of the postage stamp request API has slipped in here because we need to control
+        # the new state of the request
+        if ($action eq 'PROCESS') {
+            $newState = 'run';
+        } elsif ($action eq 'PREVIEW') {
+            $newState = 'parsed';
+        } else {
+            print STDERR "WARNING Ignoring unexpected value for ACTION found in request header: $action\n";
+        }
     } else {
         $fault = $error_code >> 8;
@@ -275 +287 @@
     # find the EXTNAME in the input fits table
     my $file_name = shift;
-    my $out = `echo $file_name | fields -x 0 EXTNAME`;
+    my $r_action = shift;
+
+    my $out = `echo $file_name | fields -x 0 EXTNAME ACTION`;
 
     if ($out) {
         # output from fields is filename value
-        my ($dummy, $extname) = split " ", $out;
+        my ($dummy, $extname, $action) = split " ", $out;
+
+        # Set the action if it is defined in the request header
+        # XXX:consider doing this only if extname is PS1_PS_REQUSET
+        $$r_action = $action if ($action);
 
         return $extname;

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_request_file

@@ -54 +54 @@
                     value => undef
         },
-        { name => 'USERNAME',
-                    writetype => TSTRING,
-                    comment => 'username for request (optional)',
-                    value => undef
-        },
+#        { name => 'USERNAME',
+#                    writetype => TSTRING,
+#                    comment => 'username for request (optional)',
+#                    value => undef
+#        },
         { name => 'EMAIL',
                     writetype => TSTRING,
@@ -97 +97 @@
         { name => 'MJD_MIN',    type => 'D',   writetype => TDOUBLE },
         { name => 'MJD_MAX',    type => 'D',   writetype => TDOUBLE },
+        # new in version 2
         { name => 'RUN_TYPE',   type => '16A', writetype => TSTRING },
+        { name => 'FWHM_MIN',   type => 'D',   writetype => TDOUBLE },
+        { name => 'FWHM_MAX',   type => 'D',   writetype => TDOUBLE },
 
         { name => 'COMMENT ',   type => '64A', writetype => TSTRING },
@@ -326 +329 @@
         pop @colData;
         pop @colData;
+        pop @colData;
+        pop @colData;
     }
 
@@ -346 +351 @@
         }
         # trim leading whitespace from comment
-        $comment =~ s/^\s+//;
+        $comment =~ s/^\s+// if $comment;
         if ($comment) {
             if ($comment =~ /#/) {

branches/eam_branches/ipp-20130307/pstamp/scripts/pstamp_webrequest.pl

@@ -5 +5 @@
 #
 # The arguments are the command line parameters for the program psmkreq
-#
-# Unless the argument -list is provided the output is the request id for the resulting request
-#
-# If -list is the first argument, the request file is built and parsed and the
-# the selected "input" uris are listed on stdout.
 #
 # Note: Despite the name there nothing particularly web specific about this program.
@@ -23 +18 @@
 
 my $host = hostname();
-my $verbose = 1;
+my $verbose = 0;
 my $dbname;
 my $dbserver;
+my $pretend;
+my $label = 'WEB';
+my $req_name_prefix = 'web';
+my $username;
 
 GetOptions(
     'dbname=s'      =>  \$dbname,
     'dbserver=s'    =>  \$dbserver,
-    'verbose'       => \$verbose,
+    'label=s'       =>  \$label,
+    'username=s'    =>  \$username,
+    'prefix=s'      =>  \$req_name_prefix,
+    'pretend'       =>  \$pretend,
+    'verbose'       =>  \$verbose,
 );
 
@@ -36 +39 @@
 
 if ($verbose) {
-    print "\n\n";
-    print "Starting script $0 on $host\n\n";
+    print STDERR "\n\n";
+    print STDERR "Starting script $0 on $host\n\n";
 }
 
@@ -70 +73 @@
 # make a request file
 my $cur_dir = getcwd();
+
 #print STDERR "cur_dir is $cur_dir\n";
 
-my $datestr = strftime "%Y%m%d", gmtime;
+my $datestr = strftime "%Y/%m/%d", gmtime;
 my $datedir = "$cur_dir/webreq/$datestr";
 if (! -e $datedir ) {
-    if (!  mkdir $datedir ) {
-        print STDERR  "failed to create working directory $datedir";
+    my $rc = system "mkdir -p $datedir";
+    if ($rc) {
+        my $status = $rc >> 8;
+        print STDERR  "failed to create working directory $datedir: $rc $status";
         exit $PS_EXIT_CONFIG_ERROR;
     }
 }
 
-my $request_name = "web_" . get_webreq_num();
+my $request_name = $req_name_prefix . '_' . get_webreq_num();
 my $request_file = "$datedir/$request_name.fits";
 {
@@ -96 +102 @@
 # Queue the request
 my $req_id = 0;
-{
+unless ($pretend) {
 
     my $command = "$pstamptool -addreq -uri $request_file -ds_id 0";
-    $command .= " -label WEB";
+    $command .= " -label $label";
+    $command .= " -username $username" if $username;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -109 +116 @@
     }
     $req_id = ${$stdout_buf}[0];
+    chomp $req_id;
 }
 

branches/eam_branches/ipp-20130307/pstamp/scripts/pstampparse.pl

@@ -27 +27 @@
 my $outdir;
 my $product;
-my $label;
+my $label = "";
 my $save_temps;
 my $no_update;
@@ -57 +57 @@
     die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
+} else {
+    $req_id = 0;
+    $outdir = "nowhere";
+    $product = "dummy";
 }
 
@@ -83 +87 @@
 $pstamptool .= " -dbserver $dbserver" if $dbserver;
 
-# list_job is a deugging mode
-$no_update = 1 if $mode eq "list_job";
+# If $mode is not queue_job we are using a debugging mode
+# do not update the database
+$no_update = 1 if $mode ne "queue_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 #
-# get the data from the extension header
+# Read the keywords from the extension header
 #
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    # fields doesn't return zero when it succeeds
-    #unless ($success) {
-    #    print STDERR @$stderr_buf;
-    #}
+
+    # note fields doesn't return zero when it succeeds.
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
-# This program shouldn't have been run if the request file is bogus.
-# No need to notify the client
-my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
-my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
-
+# pstamp_parser_run.pl would not have run this program unless the request file was ok
+my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname;
+my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver >= 2) {
+    # We have a version 2 file. Require that the new keywords be supplied. 
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    # XXX check for "valid" $email
+} else {
+    # for version 1 file the action is process and email is not used
+    $action = 'PROCESS';
+    $email = 'null';
+}
+
+print "Request Header Keywords EXTVER: $extver REQ_NAME: $req_name ACTION: $action EMAIL: $email\n";
 
 # check for duplicate request name
@@ -161 +179 @@
     # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
@@ -174 +193 @@
 
 #
-# now convert the request table to an array of metadata config docs
+# now convert the request table to an array of metadatas
 #
 
@@ -191 +210 @@
     my $dtime_request_file = gettimeofday() - $start_request_file;
     print "Time to read and parse request file: $dtime_request_file\n";
-
-}
-
-#
-# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
-# in the sense that their selection parameters will yield the same "Runs".
-# Process the groups of rows together to reduce lookup time and to potentially make multiple
-# stamps from the same ppstamp process.
-#
-my @rowList;
+}
+
+my $nRows = scalar @$rows;
+print "\n$nRows rows read from request file\n";
+
+
 my $num_jobs = 0;
 my $imageList;
 my $stage;
-my $need_magic;
 foreach my $row (@$rows) {
-    # santiy check the paramaters
+
+    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
+        # this is a big request and it came from the upload page and doesn't have a specific label assigned
+        # change it to the generic one that has lower with lower priority
+        $label = 'WEB.BIG';
+        print "\nChanging label for big WEB.UP request to $label\n";
+
+        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+
+    # validate the paramaters
     if (!checkRow($row)) {
         # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
@@ -216 +245 @@
     $row->{error_code} = 0;
 
-    if (scalar @rowList == 0) {
-        push @rowList, $row;
-        next;
-    }
-
-    my $firstRow = $rowList[0];
-    if (same_images_of_interest($firstRow, $row)) {
-        # add this row to the list and move on
-        push @rowList, $row;
-        next;
-    }
-
-    # the images of interest for this new row doesn't match the list. 
-    # process the list ...
-    $num_jobs += processRows($action, \@rowList);
-
-    # and reset the list to contain just the new row
-    @rowList = ($row);
-}
-
-# out of rows process the list
-if (scalar @rowList > 0) {
-    $num_jobs += processRows($action, \@rowList);
+    $num_jobs += processRow($action, $row);
 }
 
@@ -252 +259 @@
     my $row = shift;
 
+    # check validity of the paramters in a request specification
     # If we encounter an error for a particular row add a job with the proper fault code.
+    # also adjust some paramterers like filter and set defaults for some others
 
     my $stage = $row->{IMG_TYPE};
@@ -284 +293 @@
     }
 
-
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
@@ -314 +322 @@
         return 0;
     }
+
+    my $fwhm_min = $row->{FWHM_MIN};
+    if (!defined $fwhm_min) {
+        $row->{FWHM_MIN} = 0;
+    } elsif (!validNumber($fwhm_min)) {
+        print STDERR "$fwhm_min is not a valid FWHM_MIN\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $fwhm_max = $row->{FWHM_MAX};
+    if (!defined $fwhm_max) {
+        $row->{FWHM_MAX} = 0;
+    } elsif (!validNumber($fwhm_max)) {
+        print STDERR "$fwhm_max is not a valid FWHM_MAX\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
     my $data_group = $row->{DATA_GROUP};
     if (!defined $data_group) {
@@ -363 +389 @@
     }
 
-    # $mode list_uri is a debugging mode (it may used by the http interface)
-    # if this happens just croak
-   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-
     if ($req_type eq "bycoord") {
         if (!$skycenter) {
@@ -384 +405 @@
     }
 
+
     return 1;
 }
@@ -392 +414 @@
 
     $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRow {
+    my $action  = shift;
+    my $row     = shift;
+
+    my $num_jobs = 0;
+
+    my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PROJECT);
+        $num_jobs++;
+        return $num_jobs;
+    }
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+
+    my $req_type    = $row->{REQ_TYPE};
+    my $rownum      = $row->{ROWNUM};
+
+    # Since user can get unmagicked data "by coordinate" requests can go back in time
+    # to dredge unusable data from the "dark days"...
+    if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) {
+        # ... so unless the user sets mjd_min clamp it to 2009-04-01
+        # XXX: This value should live in the pstampProject table not be hardcoded here
+        $row->{MJD_MIN} = 54922;
+    }
+
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $row->{ID} = "" if !$row->{ID};
+    }
+    
+    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+    # request specification. An array reference is returned.
+    my $start_locate = gettimeofday();
+
+    print "\nCalling new_locate_images for row: $rownum\n";
+
+    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+
+    my $dtime_locate = gettimeofday() - $start_locate;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
+
+    my @rowList = ($row);
+    $num_jobs += queueJobs($action, \@rowList, $imageList);
+
+    # if this row slipped through without a job being added add one. 
+    if ($row->{job_num} == 0) {
+        print "row $row->{ROWNUM} produced no jobs\n";
+        print STDERR "row $row->{ROWNUM} produced no jobs\n";
+        my $error_code = $row->{error_code};
+        $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+        insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+    }
+
     return $num_jobs;
 }
@@ -401 +484 @@
 
     if ($action eq 'LIST') {
+        # LIST is not allowed by caller. Can't get here.
         return list_targets($rowList);
     }
@@ -420 +504 @@
     }
     my $req_type  = $row->{REQ_TYPE};
-    $stage        = $row->{IMG_TYPE};
+    my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
     my $component = $row->{COMPONENT};
@@ -436 +520 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    $need_magic    = $proj_hash->{need_magic};
+    my $need_magic    = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -496 +580 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -506 +590 @@
     $row->{need_magic} = $need_magic;
 
-    $num_jobs += queueJobs($mode, \@rowList, $imageList);
+    $num_jobs += queueJobs($action, $rowList, $imageList);
 
     # if a row slipped through with no jobs add a faulted one
-    foreach my $row (@rowList) {
+    foreach my $row (@$rowList) {
         if ($row->{job_num} == 0) {
             print "row $row->{ROWNUM} produced no jobs\n";
@@ -528 +612 @@
     my $image = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $rownum = $row->{ROWNUM};
@@ -611 +695 @@
     write_params($output_base, $image);
 
-    my $newState = "run";
+    my $newState = $action eq 'PROCESS' ? "run" : "parsed";
     my $fault = 0;
     my $dep_id;
 
-    queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+    queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
     my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
@@ -644 +728 @@
 sub queueJobs
 {
-    my $mode = shift;
+    my $action = shift;
     my $rowList = shift;
     my $imageList = shift;
 
-    my $firstRow = $rowList[0];
+    my $firstRow = $rowList->[0];
     my $stage    = $firstRow->{IMG_TYPE};
     my $job_type = $firstRow->{JOB_TYPE};
@@ -656 +740 @@
 
     if ($mode eq "list_uri") {
+        $num_jobs = $imageList ? scalar @$imageList : 0;
+        print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n";
         foreach my $image (@$imageList) {
             print "$image->{image}\n";
+            ++$firstRow->{job_num};
         }
     } elsif ($job_type eq "get_image") {
@@ -664 +751 @@
         my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
 
-        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $action);
 
     } else {
@@ -692 +779 @@
                 my $row = $rowList->[$i];
 
-                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $action);
             }
         }
@@ -706 +793 @@
     my $stage = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $num_jobs = 0;
@@ -753 +840 @@
         write_params($output_base, $image);
 
-        my $newState = "run";
+        my $newState = $action eq 'PROCESS' ? "run" : "parsed";
         my $fault = 0;
         my $dep_id;
 
-        queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+        queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
         $num_jobs++;
@@ -899 +986 @@
 sub get_dependent 
 {
-    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
+    my ($action, $r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
     # chipRun's can be in full state if destreaking is necessary
@@ -918 +1005 @@
     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
-
-    # compute rlabel for the run.
-    # XXX: This bit of policy shouldn't be buried so deeply in the code
-    # For now use one that implies 'postage stamp server' 'update' 'request_label"
-    my $rlabel = "ps_ud_" . $label if $label;
-    $command .= " -rlabel $rlabel" if $rlabel;
+    $command .= ' -hold' if $action eq 'PREVIEW';
+
+    if ($label) {
+        # compute rlabel for the run.
+        # XXX: This bit of policy shouldn't be buried so deeply in the code
+        # For now use one that implies 'postage stamp server' 'update' 'request_label"
+        my $rlabel = "ps_ud_" . $label;
+        $command .= " -rlabel $rlabel";
+    }
 
     if (!$no_update) {
@@ -931 +1021 @@
             my $fault = $error_code >> 8;
             print STDERR "$command failed with fault $fault\n";
-            if ($fault < 10) {
+            if ($fault < $PSTAMP_FIRST_ERROR_CODE) {
                 # pstamptool returns an error if an existing depenent is faulted
-                # Set the object to not available even if the fault < 10
+                # Set the object to not available even if the fault < $PSTAMP_FIRST_ERROR_CODE
                 # which is nominally a recoverable error in order to keep
                 # the request from faulting (which can be very expensive if
@@ -939 +1029 @@
                 $fault = $PSTAMP_NOT_AVAILABLE
             }
-            if ($fault >= 10) {
+            if ($fault >= $PSTAMP_FIRST_ERROR_CODE) {
                 $$r_dep_id = 0;
                 $$r_fault = $fault;
@@ -967 +1057 @@
 
 sub queueUpdatesIfNeeded {
+    my $action = shift;
     my $stage = shift;
     my $image = shift;
@@ -973 +1064 @@
     my $r_fault = shift;
     my $r_dep_id = shift;
+
+    my $need_magic = 0;
 
     if ($stage ne 'raw') {
@@ -1016 +1109 @@
                     # set up to queue an update run
                     my $require_magic = ($need_magic or $image->{magicked});
-                    get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
+                    get_dependent($action, \$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
                         $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
                 }
@@ -1070 +1163 @@
     my $fault = shift;
 
-    carp $msg;
+    print STDERR $msg;
 
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary