Changeset 36680 for branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl

Timestamp:

Apr 21, 2014, 5:42:34 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (20 diffs)

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+my $dump_params = 0;
 
 # set this to true to disable update processing
@@ -48 +49 @@
     'save-temps'=>  \$save_temps,
     'no-update' =>  \$no_update,
+    'dump-params' => \$dump_params,
 );
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76 +80 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116 +120 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175 +184 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212 +226 @@
 }
 
-my $nRows = scalar @$rows;
+my $nRows = $rows ? scalar @$rows : 0;
 print "\n$nRows rows read from request file\n";
 
+unless ($nRows) {
+    # pstamp_job_run was invoked so the request file must have contained a valid header
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
+    print STDERR "Invalid request file.\n";
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    exit 0;
+}
+
+
+# if label is for one of the high priority channels, watch for big requests
+my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
+
+# XXX: these should be in a configuration file somewhere not hard coded
+my $big_limit = 100;
+my $job_big_limit = 400;
+
+if ($watch_for_big_requests and $nRows > $big_limit) {
+    $label = change_to_lower_priority_label($label);
+    $watch_for_big_requests = 0;
+}
 
 my $num_jobs = 0;
-my $imageList;
-my $stage;
+
 foreach my $row (@$rows) {
-
-    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
-        # this is a big request and it came from the upload page and doesn't have a specific label assigned
-        # change it to the generic one that has lower with lower priority
-        $label = 'WEB.BIG';
-        print "\nChanging label for big WEB.UP request to $label\n";
-
-        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
 
     # validate the paramaters
@@ -246 +267 @@
 
     $num_jobs += processRow($action, $row);
+
+    # see whether number of jobs limit for high priority request was exceeded
+    if ($watch_for_big_requests and $num_jobs > $job_big_limit) {
+        $label = change_to_lower_priority_label($label);
+        $watch_for_big_requests = 0;
+    }
 }
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
 exit 0;
+
+# end of main function
+
+sub change_to_lower_priority_label {
+    my $label = shift;
+    my $old_label = $label;
+    $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+    print "\nChanging label for big $old_label request to $label\n";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    return $label;
+}
 
 sub checkRow {
@@ -267 +313 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
+        $rownum = 'NULL' if !defined $rownum;
         print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -272 +319 @@
     }
     my $job_type = $row->{JOB_TYPE};
-    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+    if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) {
+        $job_type = 'NULL' if !defined $job_type;
         print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -281 +329 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        $req_type = 'NULL' if !defined $req_type;
         print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -286 +335 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
+        # get_image jobs are quite expensive in terms of space so we are currently restricting them
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' 
+            or ($req_type eq 'byskycell' and $stage eq 'stack')
+            or ($stage eq 'stack_summary')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -295 +347 @@
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        if ($job_type eq 'get_image') {
+        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
             $row->{COMPONENT} = 'all';
         } else {
@@ -384 +436 @@
     }
 
-    if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
+    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
+        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -457 +509 @@
     my $start_locate = gettimeofday();
 
-    print "\nCalling new_locate_images for row: $rownum\n";
-
-    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+    print "\nCalling locate_images_for_row for row: $rownum\n";
+
+    my $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -521 +573 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -532 +584 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -581 +603 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
+    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -701 +723 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);
 
@@ -758 +794 @@
             print "$image->{image}\n";
             ++$firstRow->{job_num};
+            if ($dump_params) {
+                my $rownum = $firstRow->{ROWNUM};
+                my $jobnum = $firstRow->{job_num};
+                my $filter = substr $image->{filter}, 0, 1;
+                my $output_base = "${rownum}_${jobnum}_${filter}";
+                write_params($output_base, $image);
+            }
         }
     } elsif ($job_type eq "get_image") {
@@ -1163 +1206 @@
     }
     if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
-        ($img_type eq "stack") or ($img_type eq "diff")) {
+        ($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
         return 1;
     } else {