Changeset 36071 for branches/eam_branches/ipp-20130711/pstamp/scripts/pstampparse.pl

Timestamp:

Aug 30, 2013, 4:48:58 PM (13 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20130711/pstamp/scripts

Files:

• . (modified) (1 prop)
• pstampparse.pl (modified) (17 diffs)

branches/eam_branches/ipp-20130711/pstamp/scripts/pstampparse.pl

@@ -51 +51 @@
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76 +78 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116 +118 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175 +182 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212 +224 @@
 }
 
-my $nRows = scalar @$rows;
+my $nRows = $rows ? scalar @$rows : 0;
 print "\n$nRows rows read from request file\n";
+
+unless ($nRows) {
+    # pstamp_job_run was invoked so the request file must have contained a valid header
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
+    print STDERR "Invalid request file.\n";
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    exit 0;
+}
+
 
 
@@ -219 +242 @@
 my $imageList;
 my $stage;
+my $big_limit = 100;    # XXX: this should be in a configuration file some where
 foreach my $row (@$rows) {
 
-    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
-        # this is a big request and it came from the upload page and doesn't have a specific label assigned
-        # change it to the generic one that has lower with lower priority
-        $label = 'WEB.BIG';
-        print "\nChanging label for big WEB.UP request to $label\n";
+
+    if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > $big_limit or $num_jobs > $big_limit)
+        and $req_id and !$no_update) {
+
+        # This is a "big" request and it came from one of the "high priority" channels
+        # and doesn't have a specific label assigned.
+        # Change the label to a value that its jobs run with lower priority.
+
+        my $old_label = $label;
+        $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+        print "\nChanging label for big $old_label request to $label\n";
 
         my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
@@ -249 +279 @@
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
@@ -286 +318 @@
     }
     if ($job_type eq 'get_image') {
-        unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
-            print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
+        # get_image jobs are quite expensive in terms of space so we are currently restricting them
+        unless ($req_type eq 'byid' or $req_type eq 'byexp' 
+            or ($req_type eq 'byskycell' and $stage eq 'stack')
+            or ($stage eq 'stack_summary')) {
+            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
             return 0;
@@ -295 +330 @@
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
-        if ($job_type eq 'get_image') {
+        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
             $row->{COMPONENT} = 'all';
         } else {
@@ -384 +419 @@
     }
 
-    if (($req_type eq "byexp") and ($stage eq "stack")) {
-        print STDERR "byexp not implemented for stack stage. row: $rownum\n";
+    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
+        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
         return 0;
@@ -457 +492 @@
     my $start_locate = gettimeofday();
 
-    print "\nCalling new_locate_images for row: $rownum\n";
+    print "\nCalling locate_images_for_row for row: $rownum\n";
 
     $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
@@ -521 +556 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -532 +567 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -701 +706 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);
 
@@ -719 +738 @@
     if (!$no_update) {
         # mode eq "queue_job"
+        my $start_addjob = gettimeofday();
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -728 +748 @@
             my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
         }
+        my $dtime_addjob = gettimeofday() - $start_addjob;
+        print "Time to addjob for row $rownum $dtime_addjob\n";
     } else {
         print "skipping command: $command\n";
@@ -1160 +1182 @@
     }
     if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
-        ($img_type eq "stack") or ($img_type eq "diff")) {
+        ($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
         return 1;
     } else {