Changeset 36192 for branches/eam_branches/ipp-20130904/pstamp/scripts

Timestamp:

Oct 9, 2013, 4:06:09 PM (13 years ago)

Author:

eugene

Message:

merge from trunk

Location:

branches/eam_branches/ipp-20130904/pstamp/scripts

Files:

• . (modified) (1 prop)
• psmkreq (modified) (2 diffs)
• pstamp_checkdependent.pl (modified) (2 diffs)
• pstamp_job_run.pl (modified) (6 diffs)
• pstampparse.pl (modified) (8 diffs)

branches/eam_branches/ipp-20130904/pstamp/scripts/psmkreq

@@ -177 +177 @@
 $id = 0 if !$id;
 
-unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack') {
+unless ($stage eq 'raw' or $stage eq 'chip' or $stage eq 'warp' or $stage eq 'diff' or $stage eq 'stack'
+    or $stage eq 'stack_summary') {
     die "$stage is not a valid value for stage\n";
 }
@@ -208 +209 @@
         $option_mask |= $PSTAMP_SELECT_PSF      if $psf;
         $option_mask |= $PSTAMP_SELECT_BACKMDL  if $backmdl;
+        $option_mask |= $PSTAMP_SELECT_UNCOMPRESSED   if $uncompressed;
         $option_mask |= $PSTAMP_SELECT_UNCONV   if $unconvolved;
         $option_mask |= $PSTAMP_USE_IMFILE_ID   if $use_imfile_id;

branches/eam_branches/ipp-20130904/pstamp/scripts/pstamp_checkdependent.pl

@@ -304 +304 @@
             }
             $queued_update = 1;
+        } elsif ($chip->{state} eq 'cleaned' and $chip->{data_state} eq 'update') {
+            # we've had a number of runs in this limbo state
+
+            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to change ${stage}Run $stage_id $component from goto_cleaned to update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+            $queued_update = 1;
         } elsif ($chip->{fault}) {
             $fault_count++;
@@ -460 +476 @@
             print "skipping $command\n";
         }
+    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
+        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to change state of ${stage}Run $stage_id to update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+
     } elsif (scalar @chipsToUpdate > 0) {
         my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic);

branches/eam_branches/ipp-20130904/pstamp/scripts/pstamp_job_run.pl

@@ -236 +236 @@
                            $PSTAMP_SELECT_JPEG     => 'jpg',
                            $PSTAMP_SELECT_EXP      => 'exp.fits',
-                           $PSTAMP_SELECT_NUM      => 'num.fits');
+                           $PSTAMP_SELECT_NUM      => 'num.fits',
+                           $PSTAMP_SELECT_EXPJPEG  => 'exp.jpg',
+                           $PSTAMP_SELECT_NUMJPEG  => 'num.jpg');
 
         my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE 
-            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
+            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES
+            | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM 
+            | $PSTAMP_SELECT_EXPJPEG | $PSTAMP_SELECT_NUMJPEG);
+
 
         foreach my $key (keys (%extensions)) {
@@ -261 +266 @@
         close $F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
-        $jobStatus = $PSTAMP_NO_OVERLAP;
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP || $exitStatus == $PSTAMP_NO_VALID_PIXELS) {
+        $jobStatus = $exitStatus;
     } else {
         my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
@@ -382 +387 @@
         my $backmdl_file = $params->{backmdl} if ($options & $PSTAMP_SELECT_BACKMDL);
         my $pattern_file = $params->{pattern} if ($options & $PSTAMP_SELECT_BACKMDL);
+        my $filter = $params->{filter};
+        $filter = ' ' if !$filter;
+        $filter = substr $filter, 0, 1;
         my $cmf_file;
 #        if ($stage ne 'chip') {
@@ -394 +402 @@
             if (!$therest or !$rownum or !$jobnum);
 
-        my $prefix = "${rownum}_${jobnum}_";
+        # XXX: Here we are assuming the form of the output file names
+        # if we change this in pstampparse we'll need to remember ....
+        # (the last time I forgot)
+        my $prefix = "${rownum}_${jobnum}_${filter}_";
 
         if ($cmf_file) {
@@ -573 +584 @@
     $params->{exp}    = $path_base . ".exp.b1.fits";
     $params->{num}    = $path_base . ".num.b1.fits";
+    $params->{expjpeg} = $path_base . ".exp.0.b1.jpeg";
+    $params->{numjpeg} = $path_base . ".num.0.b1.jpeg";
 }
 
@@ -632 +645 @@
         if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
             $options &= ~$PSTAMP_SELECT_NUM;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_EXPJPEG) {
+        if (!myCopy("$outputBase.exp.jpg", $params->{expjpeg}, 'exp', 0)) {
+            $options &= ~$PSTAMP_SELECT_EXPJPEG;
+        }
+    }
+    if ($options & $PSTAMP_SELECT_NUMJPEG) {
+        if (!myCopy("$outputBase.num.jpg", $params->{numjpeg}, 'num', 0)) {
+            $options &= ~$PSTAMP_SELECT_NUMJPEG;
         }
     }

branches/eam_branches/ipp-20130904/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+my $dump_params = 0;
 
 # set this to true to disable update processing
@@ -48 +49 @@
     'save-temps'=>  \$save_temps,
     'no-update' =>  \$no_update,
+    'dump-params' => \$dump_params,
 );
 
@@ -238 +240 @@
 
 
+# if label is for one of the high priority channels, watch for big requests
+my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
+
+my $big_limit = 100;    # XXX: this should be in a configuration file somewhere not hard coded
+
+if ($watch_for_big_requests and $nRows > $big_limit) {
+    $label = change_to_lower_priority_label($label);
+    $watch_for_big_requests = 0;
+}
 
 my $num_jobs = 0;
-my $imageList;
-my $stage;
-my $big_limit = 100;    # XXX: this should be in a configuration file some where
+
 foreach my $row (@$rows) {
-
-
-    if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > $big_limit or $num_jobs > $big_limit)
-        and $req_id and !$no_update) {
-
-        # This is a "big" request and it came from one of the "high priority" channels
-        # and doesn't have a specific label assigned.
-        # Change the label to a value that its jobs run with lower priority.
-
-        my $old_label = $label;
-        $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
-        print "\nChanging label for big $old_label request to $label\n";
-
-        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
-        }
-    }
 
     # validate the paramaters
@@ -276 +265 @@
 
     $num_jobs += processRow($action, $row);
+
+    # see whether number of jobs limit for high priority request was exceeded
+    if ($watch_for_big_requests and $num_jobs > $big_limit) {
+        $label = change_to_lower_priority_label($label);
+        $watch_for_big_requests = 0;
+    }
 }
 
@@ -286 +281 @@
 
 exit 0;
+
+# end of main function
+
+sub change_to_lower_priority_label {
+    my $label = shift;
+    my $old_label = $label;
+    $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
+    print "\nChanging label for big $old_label request to $label\n";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    return $label;
+}
 
 sub checkRow {
@@ -494 +506 @@
     print "\nCalling locate_images_for_row for row: $rownum\n";
 
-    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+    my $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
 
     my $dtime_locate = gettimeofday() - $start_locate;
@@ -586 +598 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
+    my $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -777 +789 @@
             print "$image->{image}\n";
             ++$firstRow->{job_num};
+            if ($dump_params) {
+                my $rownum = $firstRow->{ROWNUM};
+                my $jobnum = $firstRow->{job_num};
+                my $filter = substr $image->{filter}, 0, 1;
+                my $output_base = "${rownum}_${jobnum}_${filter}";
+                write_params($output_base, $image);
+            }
         }
     } elsif ($job_type eq "get_image") {