Changeset 37403 for branches/eam_branches/ps2-tc3-20130727/pstamp/scripts

Timestamp:

Sep 19, 2014, 4:05:27 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ps2-tc3-20130727

Files:

• . (modified) (1 prop)
• pstamp/scripts/psmkreq (modified) (3 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (5 diffs, 1 prop)
• pstamp/scripts/pstamp_finish.pl (modified) (1 diff, 1 prop)
• pstamp/scripts/pstamp_insert_request.pl (modified) (1 diff)
• pstamp/scripts/pstamp_job_run.pl (modified) (6 diffs, 1 prop)
• pstamp/scripts/pstamp_save_server_status.pl (modified) (1 prop)
• pstamp/scripts/pstamp_webrequest.pl (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (1 diff, 1 prop)
• pstamp/scripts/request_finish.pl (modified) (3 diffs, 1 prop)

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq

@@ -29 +29 @@
 my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base);
 
-my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse);
+my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse, $restorebackground);
 my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait);
 
@@ -120 +120 @@
     'convolved'         => \$convolved,
     'uncompressed'      => \$uncompressed,
+    'restorebackground' => \$restorebackground,
     'use_imfile_id'     => \$use_imfile_id,
     'do_not_wait'       => \$no_wait,
@@ -224 +225 @@
         $option_mask |= $PSTAMP_USE_IMFILE_ID   if $use_imfile_id;
         $option_mask |= $PSTAMP_SELECT_INVERSE  if $inverse;
+        $option_mask |= $PSTAMP_RESTORE_BACKGROUND  if $restorebackground;
     }
     $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_wait;

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl

@@ -20 +20 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use Carp;
+
+my $disable_3PI_updates = 0;
+my $ra_max_3PI_hours = 22.5;  # disable updates for skycells with ra greater than this (hours)
+my $ra_max_3PI = $ra_max_3PI_hours * 15 * 3.14149 / 180.; # rawExp.ra is in radians
 
 # XXX: put this in a module somewhere
@@ -257 +261 @@
         push @chips, $chip->{class_id};
 
+        if ($disable_3PI_updates && ($chip->{data_group} =~ /ThreePi/)) {
+            # we are getting close to the end of PV2. Do not update PV1 chips if RA is above the limit.
+            my $ra = $chip->{ra};
+            if (!defined($ra) or ($ra > $ra_max_3PI)) {
+                print "3PI updates are currently disabled for RA $ra > $ra_max_3PI (radians)\n";
+                return $PSTAMP_NOT_AVAILABLE;
+            } else {
+                print "allowing 3PI updates RA $ra\n";
+            }
+        } 
+
         my $state = $chip->{state};
         my $data_state = $chip->{data_state};
@@ -303 +318 @@
             $command .= " -set_label $rlabel" if $rlabel;
 
-            # if this is one of the chipRuns from the PV1 LAP regenerate without using the ppImage configdump file
-            if ($chip->{data_group} =~ /^LAP.ThreePi.20130706/) {
-                $command .= ' -set_update_mode 1';
+            my $update_mode;
+            if ($chip->{data_group} =~ /^LAP.ThreePi.20120706/) {
+                # if this is one of the chipRuns from the PV1 LAP regenerate without using the ppImage configdump file.
+                # XXX: PV2 processing is complete. Don't do this anymore
+                # $update_mode = 1;
+            } elsif ($chip->{data_group} =~ /^M31.rp.2013/) {
+                # Version of M31 processed with recipes and auxiliary masks incompatible with the
+                # current code. Run from scratch which will use compatible versions of the same masks..
+                $update_mode = 1;
             } 
+            $command .= " -set_update_mode $update_mode" if $update_mode;
 
             if (!$no_update) {
@@ -371 +393 @@
     my $data_state = $metadata->{data_state};
 
+    # we are getting close to the end of PV2. optionally do not update 3Pi data.
+    if ($disable_3PI_updates && ($metadata->{data_group} =~ /ThreePi/)) {
+        my $ra = $metadata->{ra};
+        if (!defined $ra or ($ra > $ra_max_3PI)) {
+            print "3PI updates are currently disabled for RA $ra > $ra_max_3PI (radians)\n";
+            return $PSTAMP_NOT_AVAILABLE;
+        } else {
+            print "allowing 3PI updates RA $ra\n";
+        }
+    } 
+
     # The update system currently requires that data that has been magicked be destreaked
     # at chip stage so it needs magic even if the customer doesn't.
@@ -451 +484 @@
         }
         $chip_id = $chip->{chip_id};
+        # XXX: -scmap doesn't have ra and dec. Copy it from the warp
+        $chip->{ra} = $metadata->{ra};
+        $chip->{decl} = $metadata->{decl};
 
         # if chip has been magicked before require it to be magicked again

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_finish.pl

@@ -211 +211 @@
                         chomp $line;
                         my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line;
-                        # ... instead let dsreg compute the paramters by leaving them blank
-                        print $rlf "$img_name|||$filetype|\n";
+                        my $use_supplied_size = 1;
+                        if ($use_supplied_size) {
+                            print $rlf "$img_name|$reported_size|$reported_sum|$filetype|\n";
+                        } else {
+                            # ... instead let dsreg compute the paramters by leaving them blank
+                            print $rlf "$img_name|||$filetype|\n";
+                        }
 
                         # add line to the table definition file

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl

@@ -119 +119 @@
 }
 
-my $datestr = strftime "%Y%m%d", gmtime;
+# put file in directory for the current date
+my $datestr = strftime "%Y/%m/%d", gmtime;
 my $datedir = "$workdir/webreq/$datestr";
 if (! -e $datedir ) {
-    if (!  mkdir $datedir ) {
-        print STDERR  "failed to create working directory $datedir";
+    my $rc = system "mkdir -p $datedir";
+    if ($rc) {
+        my $status = $rc >> 8;
+        print STDERR  "failed to create working directory $datedir: $rc $status";
         exit $PS_EXIT_CONFIG_ERROR;
     }
 }
+
 
 my $webreq_num = get_webreq_num();

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl

@@ -29 +29 @@
 my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); 
 my ($verbose, $dbname, $dbserver, $no_update, $save_temps);
+my $very_verbose = 0; # for debugging
 
 GetOptions(
@@ -67 +68 @@
 }
 
+if ($verbose && $save_temps) {
+    # we're probably debugging turn up the verbosity
+    $very_verbose = 1;
+}
+
 if (!$dbserver) {
     $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
 }
 
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+$temproot = "/tmp" if !defined $temproot;
+
 my $missing_tools;
 
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
-my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
-my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
-my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
+my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
+my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $psgetcalibinfo = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
 my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
 my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
+my $ppBackground = can_run('ppBackground') or (warn "Can't find ppBackground" and $missing_tools = 1);
+my $ppBackgroundStack = can_run('ppBackgroundStack') or (warn "Can't find ppBackgroundStack" and $missing_tools = 1);
+my $stack_bkg_mk_mdc = can_run('stack_bkg_mk_mdc.pl') or (warn "Can't find stack_bkg_mk_mdc.pl" and $missing_tools = 1);
+my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1);
+
 
 if ($missing_tools) {
@@ -120 +134 @@
     }
 
+    if ($options & $PSTAMP_RESTORE_BACKGROUND) {
+        #  this subroutine creates a background restored version of the image or fails.
+        # upon success params->{image} is replaced with the new (temporary) version
+        my $error_code = create_background_restored_images($params);
+        if ($error_code) {
+            # if error code is one of the permanant ones set state of job to stop
+            my_die("failed to create background restored images", $job_id, $error_code, 
+                $error_code >= $PSTAMP_FIRST_ERROR_CODE ? 'stop' : undef);
+        }
+    }
+
     my $image = $params->{image};
     my $mask;
     my $variance;
+
     my $fileArgs = " -file $params->{image}";
     my @file_list = ($params->{image});
@@ -197 +223 @@
 
     # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
-    # otherwise we make copies of the input files to the outputs
+    # if stage is stack_summary we make copies of the input files to the outputs
     my $use_ppstamp = ($stage ne 'stack_summary');
 
@@ -216 +242 @@
             $exitStatus = $PS_EXIT_SYS_ERROR;
         }
-        # XXX: if stage is stack deal with EXP and NUM images if selected
+
+        # if stage is stack deal with EXP and NUM images if selected by running ppstamp again
+        if (!$exitStatus and $stage eq 'stack' and ($options & ($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM))) {
+            # XXX Here I am assuming that nothing relevant gets added to $argString 
+            # Need to check
+            my $roiArgs = $params->{job_args};
+
+            # XXX: define expnum and exptime input images in the params file so that we don't have to
+            # make assumptions about the file rules here
+            my $pathBase = $params->{stack_path_base};
+            $pathBase = $params->{path_base} unless defined $pathBase;
+            unless (defined $pathBase) {
+                my_die("stack path base undefined found when creating exp or expnum image", $job_id, $PS_EXIT_PROG_ERROR);
+            }
+            $pathBase .= ".unconv";
+            my $tmpBase = "$outputBase.TMP";
+            # nocompress because exp image gets corrupted if we do
+            my $command = "$ppstamp $tmpBase $roiArgs  -nocompress -stage stack";
+            # XXX: use file rules
+            # Image is the EXP (exptime) image
+            $command .= " -file $pathBase.exp.fits";
+            # treat number image as mask
+            $command .= " -mask $pathBase.num.fits" if ($options & $PSTAMP_SELECT_NUM);
+
+            $command .= " -forheader $calibfile" if $calibfile;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+
+            if (WIFEXITED($error_code)) {
+                $exitStatus = WEXITSTATUS($error_code);
+            } else {
+                print STDERR "ppstamp failed error_code: $error_code\n";
+                $exitStatus = $PS_EXIT_SYS_ERROR;
+            }
+
+            # command to compress fits files to stdout using gzip
+            my $fpack_gzip = "$fpack -g -S";
+            $fpack_gzip .= " -D" unless $save_temps;
+
+            if (!$exitStatus and ($options & $PSTAMP_SELECT_NUM)) {
+                # rename the num image if selected since it is not a mask
+                # Change .exp.mk to .num
+                my $tmpName = "$tmpBase.mk.fits";
+                my $numName = "$outputBase.num.fits";
+                # optionally fpack it while we're here unless user wants uncompressed images
+                unless ($options & $PSTAMP_SELECT_UNCOMPRESSED) {
+                    my $command = "$fpack_gzip $tmpName > $numName";
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+
+                    if (WIFEXITED($error_code)) {
+                        $exitStatus = WEXITSTATUS($error_code);
+                    } else {
+                        print STDERR "ppstamp failed error_code: $error_code\n";
+                        print STDERR "@$stderr_buf\n";
+                        $exitStatus = $PS_EXIT_SYS_ERROR;
+                    }
+                } elsif (!$exitStatus) {
+                    rename $tmpName, $numName 
+                        or my_die ("failed to rename $tmpName to $numName", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+                }
+
+            }
+            # fpack the exp image if not -nocompress
+            unless ($exitStatus or ($options & $PSTAMP_SELECT_UNCOMPRESSED)) {
+                my $command = "$fpack_gzip $tmpBase.fits > $outputBase.exp.fits";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+
+                if (WIFEXITED($error_code)) {
+                    $exitStatus = WEXITSTATUS($error_code);
+                } else {
+                    print STDERR "ppstamp failed error_code: $error_code\n";
+                    print STDERR "@$stderr_buf\n";
+                    $exitStatus = $PS_EXIT_SYS_ERROR;
+                }
+            } elsif (!$exitStatus) {
+                rename "$tmpBase.fits", "$outputBase.exp.fits"
+                    or my_die ("failed to rename $tmpBase.fits to $outputBase.exp.fits", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
     } else {
         $exitStatus = justCopyFiles($outputBase, \$options, $params);
@@ -665 +772 @@
 }
 
+# use ppBackground or ppBackgroundStack to produce background restored images
+sub create_background_restored_images {
+    my $params = shift;
+    my $stage = $params->{stage};
+
+    if ($stage ne 'stack' and $stage ne 'chip') {
+        # this function is only supported for chip and stack stage
+        # perhaps the parser should catch this ...
+        print STDERR "background restoration not currently supported for stage $params->{stage}\n";
+        return $PSTAMP_BG_RESTORE_NOT_AVAILABLE;
+    }
+
+    my $tempdir = tempdir("$temproot/pstamp.$job_id.XXXX", CLEANUP => !$save_temps);
+    my $imagedb = $params->{imagedb};
+    my $camera = $params->{camera};
+
+    # reinstantiate IPP Configuration using the now known camera so that we get the applicable file rules
+    $ipprc = PS::IPP::Config->new($camera);
+
+    my $input_image = $params->{image};
+    my $input_mask = $params->{mask};
+    my $input_variance = $params->{weight};
+
+    if ($stage eq 'stack') {
+        my $stack_id = $params->{stack_id};
+        my $mdcfile = "$tempdir/stk.$stack_id.bkg_input.mdc";
+        {
+            my $command = "$stack_bkg_mk_mdc --stack_id $stack_id --camera $camera --dbname $imagedb > $mdcfile";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform stack_bg_mk_mdc.pl: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+            }
+        }
+
+        my $outroot = "$tempdir/stk.$stack_id";
+
+        # XXX: get otapath from a config file or detrend system
+        my $otapath = "neb:///detrends/background_OTA_models.20140527/test_solutions";
+        {
+            my $command = "$ppBackgroundStack $outroot -input $mdcfile -image $input_image"
+               . " -OTApath $otapath" ;
+            $command .= " -mask $input_mask" if $input_mask;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $very_verbose);
+            unless ($success) {
+                # we turned off verbosity because ppBackgroundStack emits too much output
+                # XXX: perhaps use a log file and save it with the job data
+                print STDERR join "", @$stderr_buf;
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform ppBackgroundStack: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+            }
+        }
+        # it's all good change the image to be passed to ppstamp
+        $params->{image} = $ipprc->filename('PPBACKGROUND.STACK.OUTPUT', $outroot) or
+                &my_die("Unable to resolve ppBackgroundStack output file name", $job_id, $PS_EXIT_UNKNOWN_ERROR, 'run');
+
+    } elsif ($stage eq 'chip') {
+        my $input_mdl = $params->{backmdl};
+        my $chip_id = $params->{chip_id};
+        my $class_id = $params->{class_id};
+        my $outroot = "$tempdir/bgrestored.ch.$chip_id";
+        {
+            my $command = "$ppBackground $outroot -background $input_mdl -image $input_image";
+            # XXX: I think ppBackground might always require mask and variance images
+            $command .= " -mask $input_mask";
+            $command .= " -variance $input_variance";
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR join "", @$stderr_buf;
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform ppBackground: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+            }
+        }
+        $params->{image} = $ipprc->filename('PPBACKGROUND.OUTPUT', $outroot, $class_id) or
+                &my_die("Unable to resolve ppBackgroundStack output image name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+        $params->{mask} = $ipprc->filename('PPBACKGROUND.OUTPUT.MASK', $outroot, $class_id) or
+                &my_die("Unable to resolve ppBackgroundStack output mask name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+        $params->{weight} = $ipprc->filename('PPBACKGROUND.OUTPUT.VARIANCE', $outroot, $class_id) or
+                &my_die("Unable to resolve ppBackgroundStack output variance name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run');
+
+    } else {
+        # can't get here
+        &my_die( "background restoration not currently supported for stage $params->{stage}\n",
+            $job_id, $PS_EXIT_PROG_ERROR, 'stop'); ;
+    }
+
+    return 0;
+}
+
 sub my_die
 {

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_webrequest.pl

@@ -93 +93 @@
     my $command = "$psmkreq --req_name $request_name  --output $request_file @ARGV";
 
+if (0) {
+open DEBUG, ">>/tmp/pstamp.debug.log";
+print DEBUG "\ncommand is: $command\n";
+close DEBUG;
+}
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
         print STDERR @$stderr_buf;
+        print STDERR "\ncommand was: $command\n";
         die("Unable to perform psmkreq: $error_code");
     }

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl

@@ -179 +179 @@
         $label = "PY";
         $label_changed = 1;
+    } elsif ($lcname =~ /ncu/) {
+        $label = 'NCU';
+        $label_changed = 1;
+    } elsif (($lcname =~ /mpe_/) or ($lcname =~ /sd_/) or ($lcname =~ /^jk/)) {
+        $label = 'MPE';
+        $label_changed = 1;
     }
     print "Setting label for $req_name to $label\n" if $label_changed;

branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/request_finish.pl

@@ -20 +20 @@
 
 use PS::IPP::Config qw( :standard );
+use PS::IPP::PStamp::RequestFile qw( :standard );
 
 my ( $req_id, $req_name, $req_file, $req_type, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output );
+
+my $ipprc = PS::IPP::Config->new();
 
 GetOptions(
@@ -57 +60 @@
 if ($redirect_output) {
     my $logDest = "$outdir/reqfinish.$req_id.log";
-    my $ipprc = PS::IPP::Config->new();
     $ipprc->redirect_output($logDest);
+}
+
+if (!$dbserver) {
+    $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
 }
 
@@ -90 +96 @@
     }
 } else {
-    # Unknown request type.
-    # Since we don't have a req_name there's not much we can do so just the request's state to stop.
+    # Unknown request type. Stop job with fault invalid request.
     print STDERR "request  $req_id has unknown reqType $req_type\n" if $verbose;
-    my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $PS_EXIT_DATA_ERROR";
+
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $PSTAMP_INVALID_REQUEST";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;