Index: /tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35908) +++ /tags/ipp-20130712/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35909) @@ -176,5 +176,6 @@ $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, 0, 0, # fwhm cuts are not applied here - undef, undef, undef, $verbose); + undef, 0, # no cam run information + undef, undef, $verbose); return $results; @@ -322,5 +323,5 @@ 0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, getValOrZero($row->{FWHM_MIN}), getValOrZero($row->{FWHM_MAX}), - undef, $release_name, $survey, + undef, 0, $release_name, $survey, $verbose); @@ -351,4 +352,5 @@ my $fwhm_max = shift; my $selectedAstrom = shift; + my $selected_cam_id = shift; my $release_name = shift; my $survey = shift; @@ -602,5 +604,5 @@ # The image selectors are such that multiple runs my have be returned for the same exposure. # Return only the latest one. - $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose); + $images = filterRuns($stage, $choose_components, $need_magic, $images, $inverse, $verbose); } @@ -699,4 +701,5 @@ $out->{astrom} = $selectedAstrom; $out->{cam_path_base} = $selectedAstrom; + $out->{cam_id} = $selected_cam_id if $selected_cam_id; if ($selectedAstrom =~ /\.smf$/) { $out->{cam_path_base} =~ s/\.smf$//; @@ -946,5 +949,5 @@ $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $fwhm_min, $fwhm_max, - $chip->{astrom}, undef, undef, + $chip->{astrom}, $chip->{cam_id}, undef, undef, $verbose); @@ -970,5 +973,5 @@ $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $fwhm_min, $fwhm_max, - undef, $release_name, $survey, + undef, 0, $release_name, $survey, $verbose); @@ -1302,4 +1305,5 @@ $image->{astrom} = $astromFile; $image->{cam_path_base} = $camRoot; + $image->{cam_id} = $camRun->{cam_id}; return 1; @@ -1649,4 +1653,5 @@ sub filterRuns { my $stage = shift; + my $drop_duplicate_runs = shift; my $need_magic = shift; my $inputs = shift; @@ -1694,5 +1699,5 @@ } - $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); + # $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id)); # skip if we need magicked run and this one has never been magicked @@ -1702,5 +1707,5 @@ } - if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) { + if (($exp_id == $last_exp_id) and ($drop_duplicate_runs || ($run_id != $last_run_id))) { print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed; next; Index: /tags/ipp-20130712/ippTools/share/releasetool_listrelexp.sql =================================================================== --- /tags/ipp-20130712/ippTools/share/releasetool_listrelexp.sql (revision 35908) +++ /tags/ipp-20130712/ippTools/share/releasetool_listrelexp.sql (revision 35909) @@ -15,4 +15,5 @@ relExp.zpt_stdev, relExp.mcal, + relExp.ubercal_dist, relExp.path_base, relExp.registered, Index: /tags/ipp-20130712/ippTools/share/warptool_warped.sql =================================================================== --- /tags/ipp-20130712/ippTools/share/warptool_warped.sql (revision 35908) +++ /tags/ipp-20130712/ippTools/share/warptool_warped.sql (revision 35909) @@ -13,4 +13,5 @@ rawExp.decl, rawExp.exp_time, + camRun.cam_id, magicDSRun.state AS dsRun_state, IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id Index: /tags/ipp-20130712/ippTools/src/releasetool.c =================================================================== --- /tags/ipp-20130712/ippTools/src/releasetool.c (revision 35908) +++ /tags/ipp-20130712/ippTools/src/releasetool.c (revision 35909) @@ -778,5 +778,7 @@ if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC"); + psStringAppend(&query, "\nAND priority > 0 ORDER BY exp_id, priority DESC"); + } else { + psStringAppend(&query, "\nORDER BY exp_id, relexp_id DESC"); } Index: /tags/ipp-20130712/pstamp/scripts/Makefile.am =================================================================== --- /tags/ipp-20130712/pstamp/scripts/Makefile.am (revision 35908) +++ /tags/ipp-20130712/pstamp/scripts/Makefile.am (revision 35909) @@ -21,4 +21,5 @@ pstamp_get_image_job.pl \ psmkreq \ + psgetcalibinfo \ psstatus \ pstampstopfaulted \ Index: /tags/ipp-20130712/pstamp/scripts/psgetcalibinfo =================================================================== --- /tags/ipp-20130712/pstamp/scripts/psgetcalibinfo (revision 35909) +++ /tags/ipp-20130712/pstamp/scripts/psgetcalibinfo (revision 35909) @@ -0,0 +1,95 @@ +#!/bin/env perl + +# pstamp_get_calib_info.pl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +#use File::Temp qw( tempfile ); +#use File::Copy; +#use File::Basename qw(dirname); + +use PS::IPP::Metadata::Config; +# use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw( :standard ); +#use PS::IPP::PStamp::RequestFile qw( :standard ); +#use PS::IPP::PStamp::Job qw( :standard ); + +my ( $cam_id, $output, $dbname, $verbose, $save_temps); + +GetOptions( + 'cam_id=s' => \$cam_id, + 'output=s' => \$output, + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +die "usage: --cam_id --output [--dbname dbname --verbose]\n" + if !$cam_id or !$output; + +my $ipprc = PS::IPP::Config->new(); # IPP Configuration + +my $missing_tools; + +my $releasetool = can_run('releasetool') or (warn "Can't find releasetool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $relexp; +{ + my $command = "$releasetool -listrelexp -state calibrated -priority_order -limit 1 -cam_id $cam_id"; + $command .= " -dbname $dbname" if $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + die("Unable to perform $command error code: $error_code"); + } + my $data = join "", @$stdout_buf; + if ($data) { + # print STDERR $data if $verbose; + + my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc"); + + # no need to use parse_md_fast here + my $results = parse_md_list($metadata); + if (scalar @$results != 1) { + print STDERR "get_job_params: failed to parse_md_list\n"; + exit 0; + } + + $relexp = $results->[0]; + } +} +if (!$relexp) { + print "Failed to find calibration information for camRun: $cam_id\n"; + exit 0; +} + +open OUT, ">$output" or die "failed to open $output for writing\n"; + +print OUT "ZPCALIB F32 $relexp->{zpt_obs} # calibrated zero point\n"; +print OUT "ZPCALERR F32 $relexp->{zpt_stdev} # calibrated zero point error - 0 for ubercal exposure\n"; +print OUT "MCAL F32 $relexp->{mcal} # zero point offset due to clouds\n"; +print OUT "DVOFLAGS U32 $relexp->{flags} # dvo flags - 512 is ubercal bit\n"; +print OUT "UCALDIST S32 $relexp->{ubercal_dist} # distance to ubercal image\n"; +print OUT "DVODB STR $relexp->{dvodb} # dvo database used for calibration \n"; +print OUT "UCALFILE STR $relexp->{ubercal_file} # ubercal file used for calibration\n"; +print OUT "CALDATE STR $relexp->{time_stamp} # time of calibration\n"; +print OUT "PSREL STR $relexp->{release_name} # PS release name\n"; + +close OUT or die "failed to close $output\n"; + Index: /tags/ipp-20130712/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /tags/ipp-20130712/pstamp/scripts/pstamp_job_run.pl (revision 35908) +++ /tags/ipp-20130712/pstamp/scripts/pstamp_job_run.pl (revision 35909) @@ -78,7 +78,6 @@ my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); +my $psgetcalibinfo = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1); my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1); -my $streaksreplace = can_run('streaksreplace') or (warn "Can't find streaksreplace" and $missing_tools = 1); -my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1); @@ -100,14 +99,4 @@ # XXX: should we do any other sanity checking? my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; - - my $nan_masked = 1; - my $muggle = 0; - if (!$params->{magicked}) { - $nan_masked = 0; - } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) { - # Attempt to find or create a muggle image - $nan_masked = 0; - $muggle = 1; - } if ($stage eq "raw") { @@ -122,5 +111,5 @@ my @file_list = ($params->{image}); - if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) { + if ($options & $PSTAMP_SELECT_MASK) { $mask = $params->{mask}; $fileArgs .= " -mask $mask"; @@ -162,84 +151,35 @@ # find our output directory my $outdir = dirname($outputBase); - my ($tmpImage, $tmpMask, $tmpVariance, $tmproot); - - if ($muggle) { - # first see if the original uncensored images are around - if (check_for_backups($params, \$fileArgs)) { - # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images - print "Making stamps from backup images\n"; - } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) { - # user required uncensored but since stage isn't chip we can't rebuild them - my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop'); - } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) { - # we can only restore pixels for chip stage images if the data has been updated. - # the data will have been updated if the params->{state} the state when the job was queued is not 'full' ( - # XXX: we should probably be looking explicitly at the job and checking for a dep_id - print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n"; - # make stamps from uncensored images - $muggle = 0; + + my ($calib_fd, $calibfile); + if ($stage eq 'chip' or $stage eq 'warp') { + my $cam_id = $params->{cam_id}; + if (!$cam_id) { + carp "no cam_id found in job params\n"; + exit $PS_EXIT_PROG_ERROR; + } + ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps); + close $calib_fd; + + my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + my $exitStatus; + if (WIFEXITED($error_code)) { + $exitStatus = WEXITSTATUS($error_code); } else { - # Try and replace the streaks from the recovery images - - my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR); - $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps); - my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot"; - # find the "directory" of the input path_base - my $inputdir = dirname($image); - my $base = basename($image); - $tmpImage = "$tmproot/$base"; - - @file_list = (); - - # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and - # stage work. Import the rule here. - my $recImage = "$inputdir/REC_$base"; - my $newFileArgs = " -file $tmpImage"; - $muggle_command .= " -image $image -recimage $recImage"; - push @file_list, $recImage; - - if ($mask) { - $base = basename($mask); - $tmpMask = "$tmproot/$base"; - $newFileArgs .= " -mask $tmpMask"; - my $recMask = "$inputdir/REC_$base"; - $muggle_command .= " -mask $mask -recmask $recMask"; - push @file_list, $recMask; - } - - if ($variance) { - $base = basename($variance); - $tmpVariance = "$tmproot/$base"; - $newFileArgs .= " -variance $tmpVariance"; - my $recVariance = "$inputdir/REC_$base"; - $muggle_command .= " -weight $variance -recweight $recVariance"; - push @file_list, $recVariance; - } - if (check_files(0, @file_list)) { - # recovery files exist and are accessible restore the excised pixels - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $muggle_command, verbose => $verbose); - unless ($success) { - my $exitStatus = WEXITSTATUS($error_code); - my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus); - } - - # set the ppstamp file arguments - $fileArgs = $newFileArgs; - } else { - if ($options & $PSTAMP_REQUIRE_UNCENSORED) { - my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE); - } - # just make stamps from the censored images - print "Unable to restore uncensored images, will extract stamps from censored images\n"; - # these files won't be used so zap them - ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef); - } - } - } - - # MAGIC IS DEAD - $nan_masked = 0; + print STDERR "psgetcalibinfo failed error_code: $error_code\n"; + $exitStatus = $PS_EXIT_SYS_ERROR; + } + exit $exitStatus if $exitStatus; + + if (-s $calibfile == 0) { + print "no calibration information found for $cam_id\n"; + $calibfile = undef; + } + } + + # my ($tmpImage, $tmpMask, $tmpVariance, $tmproot); my $command = "$ppstamp $outputBase $argString $fileArgs"; @@ -247,7 +187,5 @@ $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED); $command .= " -stage $stage"; - $command .= " -censor_masked" if $nan_masked; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; + $command .= " -forheader $calibfile" if $calibfile; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -523,38 +461,4 @@ } -sub check_for_backups { - my $params = shift; - my $r_fileArgs = shift; - - my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}"; - my $results = runToolAndParse($command, $verbose); - my $dsComponent = $results->[0]; - - if ($dsComponent and ($dsComponent->{data_state} eq 'full')) { - - print "magicDSFile state is full. Backup images should exist\n"; - - # replace the file names with the backup paths - # fileArgs has the form: -file imagename [-mask maskname] [-weight weightname] - my @args = split " ", $$r_fileArgs; - - my $newFileArgs; - while (@args) { - my $a = shift @args; - my $f = shift @args; - my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f; - - my $backup_image_name = $ipprc->destreaked_filename($f); - my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name; - $newFileArgs .= " $a $backup_image_name"; - } - $$r_fileArgs = $newFileArgs; - - return 1; - } - - return 0; -} - my $neb; sub storage_object_exists Index: /tags/ipp-20130712/pstamp/src/ppstampArguments.c =================================================================== --- /tags/ipp-20130712/pstamp/src/ppstampArguments.c (revision 35908) +++ /tags/ipp-20130712/pstamp/src/ppstampArguments.c (revision 35909) @@ -26,4 +26,5 @@ fprintf(stderr, " [-variance var_image] : variance image\n"); fprintf(stderr, " [-sources sources] : sources cmf (ignored for chip stage)\n"); + fprintf(stderr, " [-forheader mdcfile : metadata config file contaiing keywords to add to fits headers\n"); fprintf(stderr, " [-stage stage] : stage of input image (raw, chip, warp, stack, diff)\n"); fprintf(stderr, " [-write_jpeg] : write a JPEG format of the image stamp\n"); @@ -135,4 +136,22 @@ usage(); } + + if ((argnum = psArgumentGet(argc, argv, "-forheader"))) { + psArgumentRemove(argnum, &argc, argv); + if (argnum >= argc) { + psError(PSTAMP_ERR_ARGUMENTS, true, "metadata config filename value required with option -forheader"); + usage(); + } + + const char *filename = argv[argnum]; + psArgumentRemove(argnum, &argc, argv); + + unsigned int nFail = 0; + options->headerAdditions = psMetadataConfigRead(NULL, &nFail, filename, false); + if (nFail || !options->headerAdditions) { + psError(PS_ERR_UNKNOWN, false, "failed to read metadata from %s", filename); + return false; + } + } if ((argnum = psArgumentGet(argc, argv, "-no_censor_masked"))) { Index: /tags/ipp-20130712/pstamp/src/ppstampMakeStamp.c =================================================================== --- /tags/ipp-20130712/pstamp/src/ppstampMakeStamp.c (revision 35908) +++ /tags/ipp-20130712/pstamp/src/ppstampMakeStamp.c (revision 35909) @@ -165,4 +165,13 @@ ppstampVersionMetadata(outHDU->header, options); + + // copy any user supplied keywords + if (options->headerAdditions) { + if (!psMetadataOverlay(outHDU->header, options->headerAdditions)) { + psError(PS_ERR_UNKNOWN, false, "Failed to copy header additions to output\n"); + return false; + } + } + return true; Index: /tags/ipp-20130712/pstamp/src/ppstampOptions.c =================================================================== --- /tags/ipp-20130712/pstamp/src/ppstampOptions.c (revision 35908) +++ /tags/ipp-20130712/pstamp/src/ppstampOptions.c (revision 35909) @@ -10,4 +10,5 @@ { psFree(options->chipName); + psFree(options->headerAdditions); } @@ -30,4 +31,5 @@ options->cellName = NULL; options->stage = NULL; + options->headerAdditions = NULL; options->censorMasked = false; options->writeJPEG = false; Index: /tags/ipp-20130712/pstamp/src/ppstampOptions.h =================================================================== --- /tags/ipp-20130712/pstamp/src/ppstampOptions.h (revision 35908) +++ /tags/ipp-20130712/pstamp/src/ppstampOptions.h (revision 35909) @@ -12,4 +12,5 @@ psString cellName; psString stage; + psMetadata *headerAdditions; bool censorMasked; bool writeJPEG;