Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/psmkreq (revision 37403) @@ -29,5 +29,5 @@ my ($ra, $dec, $x, $y, $list, $output, $req_name, $req_name_base); -my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse); +my ($image, $mask, $variance, $jpeg, $cmf, $psf, $backmdl, $inverse, $restorebackground); my ($convolved, $unconvolved, $uncompressed, $use_imfile_id, $no_wait); @@ -120,4 +120,5 @@ 'convolved' => \$convolved, 'uncompressed' => \$uncompressed, + 'restorebackground' => \$restorebackground, 'use_imfile_id' => \$use_imfile_id, 'do_not_wait' => \$no_wait, @@ -224,4 +225,5 @@ $option_mask |= $PSTAMP_USE_IMFILE_ID if $use_imfile_id; $option_mask |= $PSTAMP_SELECT_INVERSE if $inverse; + $option_mask |= $PSTAMP_RESTORE_BACKGROUND if $restorebackground; } $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_wait; Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_checkdependent.pl (revision 37403) @@ -20,4 +20,8 @@ use PS::IPP::PStamp::RequestFile qw( :standard ); use Carp; + +my $disable_3PI_updates = 0; +my $ra_max_3PI_hours = 22.5; # disable updates for skycells with ra greater than this (hours) +my $ra_max_3PI = $ra_max_3PI_hours * 15 * 3.14149 / 180.; # rawExp.ra is in radians # XXX: put this in a module somewhere @@ -257,4 +261,15 @@ push @chips, $chip->{class_id}; + if ($disable_3PI_updates && ($chip->{data_group} =~ /ThreePi/)) { + # we are getting close to the end of PV2. Do not update PV1 chips if RA is above the limit. + my $ra = $chip->{ra}; + if (!defined($ra) or ($ra > $ra_max_3PI)) { + print "3PI updates are currently disabled for RA $ra > $ra_max_3PI (radians)\n"; + return $PSTAMP_NOT_AVAILABLE; + } else { + print "allowing 3PI updates RA $ra\n"; + } + } + my $state = $chip->{state}; my $data_state = $chip->{data_state}; @@ -303,8 +318,15 @@ $command .= " -set_label $rlabel" if $rlabel; - # if this is one of the chipRuns from the PV1 LAP regenerate without using the ppImage configdump file - if ($chip->{data_group} =~ /^LAP.ThreePi.20130706/) { - $command .= ' -set_update_mode 1'; + my $update_mode; + if ($chip->{data_group} =~ /^LAP.ThreePi.20120706/) { + # if this is one of the chipRuns from the PV1 LAP regenerate without using the ppImage configdump file. + # XXX: PV2 processing is complete. Don't do this anymore + # $update_mode = 1; + } elsif ($chip->{data_group} =~ /^M31.rp.2013/) { + # Version of M31 processed with recipes and auxiliary masks incompatible with the + # current code. Run from scratch which will use compatible versions of the same masks.. + $update_mode = 1; } + $command .= " -set_update_mode $update_mode" if $update_mode; if (!$no_update) { @@ -371,4 +393,15 @@ my $data_state = $metadata->{data_state}; + # we are getting close to the end of PV2. optionally do not update 3Pi data. + if ($disable_3PI_updates && ($metadata->{data_group} =~ /ThreePi/)) { + my $ra = $metadata->{ra}; + if (!defined $ra or ($ra > $ra_max_3PI)) { + print "3PI updates are currently disabled for RA $ra > $ra_max_3PI (radians)\n"; + return $PSTAMP_NOT_AVAILABLE; + } else { + print "allowing 3PI updates RA $ra\n"; + } + } + # The update system currently requires that data that has been magicked be destreaked # at chip stage so it needs magic even if the customer doesn't. @@ -451,4 +484,7 @@ } $chip_id = $chip->{chip_id}; + # XXX: -scmap doesn't have ra and dec. Copy it from the warp + $chip->{ra} = $metadata->{ra}; + $chip->{decl} = $metadata->{decl}; # if chip has been magicked before require it to be magicked again Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_finish.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_finish.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_finish.pl (revision 37403) @@ -211,6 +211,11 @@ chomp $line; my ($img_name, $reported_size, $reported_sum, $filetype) = split '\|', $line; - # ... instead let dsreg compute the paramters by leaving them blank - print $rlf "$img_name|||$filetype|\n"; + my $use_supplied_size = 1; + if ($use_supplied_size) { + print $rlf "$img_name|$reported_size|$reported_sum|$filetype|\n"; + } else { + # ... instead let dsreg compute the paramters by leaving them blank + print $rlf "$img_name|||$filetype|\n"; + } # add line to the table definition file Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_insert_request.pl (revision 37403) @@ -119,12 +119,16 @@ } -my $datestr = strftime "%Y%m%d", gmtime; +# put file in directory for the current date +my $datestr = strftime "%Y/%m/%d", gmtime; my $datedir = "$workdir/webreq/$datestr"; if (! -e $datedir ) { - if (! mkdir $datedir ) { - print STDERR "failed to create working directory $datedir"; + my $rc = system "mkdir -p $datedir"; + if ($rc) { + my $status = $rc >> 8; + print STDERR "failed to create working directory $datedir: $rc $status"; exit $PS_EXIT_CONFIG_ERROR; } } + my $webreq_num = get_webreq_num(); Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_job_run.pl (revision 37403) @@ -29,4 +29,5 @@ my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options); my ($verbose, $dbname, $dbserver, $no_update, $save_temps); +my $very_verbose = 0; # for debugging GetOptions( @@ -67,16 +68,29 @@ } +if ($verbose && $save_temps) { + # we're probably debugging turn up the verbosity + $very_verbose = 1; +} + if (!$dbserver) { $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); } +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); +$temproot = "/tmp" if !defined $temproot; + my $missing_tools; my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); -my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); -my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); -my $psgetcalibinfo = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1); +my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); +my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); +my $psgetcalibinfo = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1); my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1); my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1); +my $ppBackground = can_run('ppBackground') or (warn "Can't find ppBackground" and $missing_tools = 1); +my $ppBackgroundStack = can_run('ppBackgroundStack') or (warn "Can't find ppBackgroundStack" and $missing_tools = 1); +my $stack_bkg_mk_mdc = can_run('stack_bkg_mk_mdc.pl') or (warn "Can't find stack_bkg_mk_mdc.pl" and $missing_tools = 1); +my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); + if ($missing_tools) { @@ -120,7 +134,19 @@ } + if ($options & $PSTAMP_RESTORE_BACKGROUND) { + # this subroutine creates a background restored version of the image or fails. + # upon success params->{image} is replaced with the new (temporary) version + my $error_code = create_background_restored_images($params); + if ($error_code) { + # if error code is one of the permanant ones set state of job to stop + my_die("failed to create background restored images", $job_id, $error_code, + $error_code >= $PSTAMP_FIRST_ERROR_CODE ? 'stop' : undef); + } + } + my $image = $params->{image}; my $mask; my $variance; + my $fileArgs = " -file $params->{image}"; my @file_list = ($params->{image}); @@ -197,5 +223,5 @@ # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile) - # otherwise we make copies of the input files to the outputs + # if stage is stack_summary we make copies of the input files to the outputs my $use_ppstamp = ($stage ne 'stack_summary'); @@ -216,5 +242,86 @@ $exitStatus = $PS_EXIT_SYS_ERROR; } - # XXX: if stage is stack deal with EXP and NUM images if selected + + # if stage is stack deal with EXP and NUM images if selected by running ppstamp again + if (!$exitStatus and $stage eq 'stack' and ($options & ($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM))) { + # XXX Here I am assuming that nothing relevant gets added to $argString + # Need to check + my $roiArgs = $params->{job_args}; + + # XXX: define expnum and exptime input images in the params file so that we don't have to + # make assumptions about the file rules here + my $pathBase = $params->{stack_path_base}; + $pathBase = $params->{path_base} unless defined $pathBase; + unless (defined $pathBase) { + my_die("stack path base undefined found when creating exp or expnum image", $job_id, $PS_EXIT_PROG_ERROR); + } + $pathBase .= ".unconv"; + my $tmpBase = "$outputBase.TMP"; + # nocompress because exp image gets corrupted if we do + my $command = "$ppstamp $tmpBase $roiArgs -nocompress -stage stack"; + # XXX: use file rules + # Image is the EXP (exptime) image + $command .= " -file $pathBase.exp.fits"; + # treat number image as mask + $command .= " -mask $pathBase.num.fits" if ($options & $PSTAMP_SELECT_NUM); + + $command .= " -forheader $calibfile" if $calibfile; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (WIFEXITED($error_code)) { + $exitStatus = WEXITSTATUS($error_code); + } else { + print STDERR "ppstamp failed error_code: $error_code\n"; + $exitStatus = $PS_EXIT_SYS_ERROR; + } + + # command to compress fits files to stdout using gzip + my $fpack_gzip = "$fpack -g -S"; + $fpack_gzip .= " -D" unless $save_temps; + + if (!$exitStatus and ($options & $PSTAMP_SELECT_NUM)) { + # rename the num image if selected since it is not a mask + # Change .exp.mk to .num + my $tmpName = "$tmpBase.mk.fits"; + my $numName = "$outputBase.num.fits"; + # optionally fpack it while we're here unless user wants uncompressed images + unless ($options & $PSTAMP_SELECT_UNCOMPRESSED) { + my $command = "$fpack_gzip $tmpName > $numName"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (WIFEXITED($error_code)) { + $exitStatus = WEXITSTATUS($error_code); + } else { + print STDERR "ppstamp failed error_code: $error_code\n"; + print STDERR "@$stderr_buf\n"; + $exitStatus = $PS_EXIT_SYS_ERROR; + } + } elsif (!$exitStatus) { + rename $tmpName, $numName + or my_die ("failed to rename $tmpName to $numName", $job_id, $PS_EXIT_UNKNOWN_ERROR); + } + + } + # fpack the exp image if not -nocompress + unless ($exitStatus or ($options & $PSTAMP_SELECT_UNCOMPRESSED)) { + my $command = "$fpack_gzip $tmpBase.fits > $outputBase.exp.fits"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (WIFEXITED($error_code)) { + $exitStatus = WEXITSTATUS($error_code); + } else { + print STDERR "ppstamp failed error_code: $error_code\n"; + print STDERR "@$stderr_buf\n"; + $exitStatus = $PS_EXIT_SYS_ERROR; + } + } elsif (!$exitStatus) { + rename "$tmpBase.fits", "$outputBase.exp.fits" + or my_die ("failed to rename $tmpBase.fits to $outputBase.exp.fits", $job_id, $PS_EXIT_UNKNOWN_ERROR); + } + } } else { $exitStatus = justCopyFiles($outputBase, \$options, $params); @@ -665,4 +772,97 @@ } +# use ppBackground or ppBackgroundStack to produce background restored images +sub create_background_restored_images { + my $params = shift; + my $stage = $params->{stage}; + + if ($stage ne 'stack' and $stage ne 'chip') { + # this function is only supported for chip and stack stage + # perhaps the parser should catch this ... + print STDERR "background restoration not currently supported for stage $params->{stage}\n"; + return $PSTAMP_BG_RESTORE_NOT_AVAILABLE; + } + + my $tempdir = tempdir("$temproot/pstamp.$job_id.XXXX", CLEANUP => !$save_temps); + my $imagedb = $params->{imagedb}; + my $camera = $params->{camera}; + + # reinstantiate IPP Configuration using the now known camera so that we get the applicable file rules + $ipprc = PS::IPP::Config->new($camera); + + my $input_image = $params->{image}; + my $input_mask = $params->{mask}; + my $input_variance = $params->{weight}; + + if ($stage eq 'stack') { + my $stack_id = $params->{stack_id}; + my $mdcfile = "$tempdir/stk.$stack_id.bkg_input.mdc"; + { + my $command = "$stack_bkg_mk_mdc --stack_id $stack_id --camera $camera --dbname $imagedb > $mdcfile"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stack_bg_mk_mdc.pl: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + } + } + + my $outroot = "$tempdir/stk.$stack_id"; + + # XXX: get otapath from a config file or detrend system + my $otapath = "neb:///detrends/background_OTA_models.20140527/test_solutions"; + { + my $command = "$ppBackgroundStack $outroot -input $mdcfile -image $input_image" + . " -OTApath $otapath" ; + $command .= " -mask $input_mask" if $input_mask; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $very_verbose); + unless ($success) { + # we turned off verbosity because ppBackgroundStack emits too much output + # XXX: perhaps use a log file and save it with the job data + print STDERR join "", @$stderr_buf; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppBackgroundStack: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + } + } + # it's all good change the image to be passed to ppstamp + $params->{image} = $ipprc->filename('PPBACKGROUND.STACK.OUTPUT', $outroot) or + &my_die("Unable to resolve ppBackgroundStack output file name", $job_id, $PS_EXIT_UNKNOWN_ERROR, 'run'); + + } elsif ($stage eq 'chip') { + my $input_mdl = $params->{backmdl}; + my $chip_id = $params->{chip_id}; + my $class_id = $params->{class_id}; + my $outroot = "$tempdir/bgrestored.ch.$chip_id"; + { + my $command = "$ppBackground $outroot -background $input_mdl -image $input_image"; + # XXX: I think ppBackground might always require mask and variance images + $command .= " -mask $input_mask"; + $command .= " -variance $input_variance"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR join "", @$stderr_buf; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppBackground: $error_code", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + } + } + $params->{image} = $ipprc->filename('PPBACKGROUND.OUTPUT', $outroot, $class_id) or + &my_die("Unable to resolve ppBackgroundStack output image name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + $params->{mask} = $ipprc->filename('PPBACKGROUND.OUTPUT.MASK', $outroot, $class_id) or + &my_die("Unable to resolve ppBackgroundStack output mask name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + $params->{weight} = $ipprc->filename('PPBACKGROUND.OUTPUT.VARIANCE', $outroot, $class_id) or + &my_die("Unable to resolve ppBackgroundStack output variance name", $job_id, $PS_EXIT_CONFIG_ERROR, 'run'); + + } else { + # can't get here + &my_die( "background restoration not currently supported for stage $params->{stage}\n", + $job_id, $PS_EXIT_PROG_ERROR, 'stop'); ; + } + + return 0; +} + sub my_die { Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_webrequest.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_webrequest.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstamp_webrequest.pl (revision 37403) @@ -93,8 +93,15 @@ my $command = "$psmkreq --req_name $request_name --output $request_file @ARGV"; +if (0) { +open DEBUG, ">>/tmp/pstamp.debug.log"; +print DEBUG "\ncommand is: $command\n"; +close DEBUG; +} + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { print STDERR @$stderr_buf; + print STDERR "\ncommand was: $command\n"; die("Unable to perform psmkreq: $error_code"); } Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/pstampparse.pl (revision 37403) @@ -179,4 +179,10 @@ $label = "PY"; $label_changed = 1; + } elsif ($lcname =~ /ncu/) { + $label = 'NCU'; + $label_changed = 1; + } elsif (($lcname =~ /mpe_/) or ($lcname =~ /sd_/) or ($lcname =~ /^jk/)) { + $label = 'MPE'; + $label_changed = 1; } print "Setting label for $req_name to $label\n" if $label_changed; Index: branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/request_finish.pl =================================================================== --- branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/request_finish.pl (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/pstamp/scripts/request_finish.pl (revision 37403) @@ -20,6 +20,9 @@ use PS::IPP::Config qw( :standard ); +use PS::IPP::PStamp::RequestFile qw( :standard ); my ( $req_id, $req_name, $req_file, $req_type, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output ); + +my $ipprc = PS::IPP::Config->new(); GetOptions( @@ -57,6 +60,9 @@ if ($redirect_output) { my $logDest = "$outdir/reqfinish.$req_id.log"; - my $ipprc = PS::IPP::Config->new(); $ipprc->redirect_output($logDest); +} + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); } @@ -90,8 +96,8 @@ } } else { - # Unknown request type. - # Since we don't have a req_name there's not much we can do so just the request's state to stop. + # Unknown request type. Stop job with fault invalid request. print STDERR "request $req_id has unknown reqType $req_type\n" if $verbose; - my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $PS_EXIT_DATA_ERROR"; + + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $PSTAMP_INVALID_REQUEST"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver;