Changeset 36061

Timestamp:

Aug 30, 2013, 3:20:12 PM (13 years ago)

Author:

bills

Message:

Include filter in pstampJob.outputBase
remove some dead magic code including references to UNCENSORED variables
only change the label to xxx.BIG once

File:

• trunk/pstamp/scripts/pstampparse.pl (modified) (10 diffs)

trunk/pstamp/scripts/pstampparse.pl

@@ -51 +51 @@
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
-die "--file is required"     if !defined($request_file_name);
+die "--file is required"   unless defined($request_file_name);
 
 if ($mode ne "list_uri") {
-    die "req_id is required"   if !$req_id;
+    die "req_id is required"  if !$req_id;
     die "outdir is required"  if !$outdir;
-    die "product is required"  if !$product;
+    die "product is required" if !$product;
 } else {
-    $req_id = 0;
-    $outdir = "nowhere";
+    # mode eq 'list_uri' (used for parser testing)
+    # these values won't be used for anything but should be defined to avoid errors
+    $req_id  = 0;
+    $outdir  = "nowhere";
     $product = "dummy";
 }
@@ -76 +78 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -116 +118 @@
 if ($extver >= 2) {
     # We have a version 2 file. Require that the new keywords be supplied. 
-    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
-
-    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
-
-    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) 
+        unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
+        $PSTAMP_INVALID_REQUEST)
+        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
+        unless $email;
+
     # XXX check for "valid" $email
 } else {
     # for version 1 file the action is process and email is not used
     $action = 'PROCESS';
-    $email = 'null';
+    $email  = 'null';
 }
 
@@ -175 +182 @@
 
 
-if ($req_id and !$no_update) {
+{
     # update the database with the request name. This will be used as the
-    # the output data store's product name
+    # the fileset name in the output data store
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
     $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
 if ($duplicate_req_name) {
     exit 0;
@@ -212 +224 @@
 }
 
-if (!$rows) {
+my $nRows = $rows ? scalar @$rows : 0;
+print "\n$nRows rows read from request file\n";
+
+unless ($nRows) {
     # pstamp_job_run was invoked so the request file must have contained a valid header
-    # We can only assume that the file is invalid or more likely empty. The print above will let the
-    # log file know. Insert a faulted fake job and exit successfully.
+    # a request file with no rows is invalid.
+    # The print above will let the log file know a bit more.
+    # Insert a faulted fake job and exit this program successfully.
     print STDERR "Invalid request file.\n";
     insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
@@ -222 +238 @@
 
 
-my $nRows = scalar @$rows;
-print "\n$nRows rows read from request file\n";
-
 
 my $num_jobs = 0;
 my $imageList;
 my $stage;
+my $big_limit = 100;    # XXX: this should be in a configuration file some where
 foreach my $row (@$rows) {
 
-    if (($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > 200 or $num_jobs > 200) and $req_id and !$no_update) {
-        # this is a big request and it came from one of the "high priority" channels
-        # upload page and doesn't have a specific label assigned
-        # change it to a generic one that runs with lower priority
+
+    if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > $big_limit or $num_jobs > $big_limit)
+        and $req_id and !$no_update) {
+
+        # This is a "big" request and it came from one of the "high priority" channels
+        # and doesn't have a specific label assigned.
+        # Change the label to a value that its jobs run with lower priority.
+
+        my $old_label = $label;
         $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
-        print "\nChanging label for big WEB.UP request to $label\n";
+        print "\nChanging label for big $old_label request to $label\n";
 
         my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
@@ -260 +279 @@
 
 if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+    # this should not happen. The function above is required to insert a fake job for any
+    # rows that did not yield any jobs.
+    print STDERR "ERROR: zero jobs created for $req_name\n";
+    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
 }
 
@@ -535 +556 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    my $need_magic    = $proj_hash->{need_magic};
+    my $need_magic = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -546 +567 @@
     $need_magic = 0 if $stage eq 'stack';
 
-    if ($need_magic) {
-
-        # this project requires that postage stamps be extracted from destreaked images
-
-        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
-            # The user has requested uncensored stamps
-
-            if (!$dest_requires_magic) {
-                # and this user's data store destination is allowed uncensored stamps, so accept the request
-                $need_magic = 0;
-            } else {
-                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
-                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
-                    # user required uncensored stamps. Can't do it so fail.
-                    foreach my $r (@$rowList) {
-                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
-                        $num_jobs++;
-                    }
-                    return $num_jobs;
-                }
-
-                # user will accept censored stamps. alter OPTION_MASK and continue
-
-                print STDERR "    Will attempt to make destreaked stamps\n";
-                # zap the offending bit in the option mask
-                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
-                foreach my $r (@$rowList) {
-                    $r->{OPTION_MASK} = $option_mask;
-                }
-            }
-        }
-    }
-    
+    # XXX: magic is dead
+    $need_magic = 0;
+
     my $numRows = scalar @$rowList;
 
@@ -715 +706 @@
     }
     $base =~ s/.fits$//;
+
+    my $filter = $image->{filter};
+    if (!$filter) {
+        if ($stage eq 'diff') {
+            $filter = $image->{filter_1};
+        }
+        if (!$filter) {
+            # XXX: perhaps this should be a programming error...
+            print STDERR "missing filter using 'X'\n";
+            $filter = 'X';
+        }
+    }
+    # use first character of filter
+    $filter = substr($filter, 0, 1);
             
-    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
     write_params($output_base, $image);