Index: trunk/ippScripts/scripts/background_chip.pl =================================================================== --- trunk/ippScripts/scripts/background_chip.pl (revision 35117) +++ trunk/ippScripts/scripts/background_chip.pl (revision 35118) @@ -14,4 +14,5 @@ use DateTime; my $mjd_start = DateTime->now->mjd; # MJD of starting script +use File::Basename; use vars qw( $VERSION ); @@ -123,4 +124,5 @@ my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id); my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id); my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id); @@ -148,7 +150,11 @@ } +# XXX: M31 TEST turn off restoring pattern correction +$apply_pattern = 0; + # Set up files &my_die("Couldn't find input file: $in_image\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_image); &my_die("Couldn't find input file: $in_mask\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_mask); +&my_die("Couldn't find input file: $in_wt\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_wt); &my_die("Couldn't find input file: $in_bg\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_bg) or !$apply_bg; &my_die("Couldn't find input file: $in_pattern\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_pattern) or !$apply_pattern; @@ -163,7 +169,76 @@ my $out_image = prepare_output("PPBACKGROUND.OUTPUT", $outroot, $class_id, 1); my $out_mask = prepare_output("PPBACKGROUND.OUTPUT.MASK", $outroot, $class_id, 1); +my $out_wt = prepare_output("PPBACKGROUND.OUTPUT.VARIANCE", $outroot, $class_id, 1); my $out_stats = prepare_output("PPBACKGROUND.STATS", $outroot, $class_id, 1); my $out_config = prepare_output("PPBACKGROUND.CONFIG", $outroot, $class_id, 1); my $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1); + +my $auxmask; + +my $apply_auxiliary_mask = 1; +if ($apply_auxiliary_mask) { + my @auxmask_start_date = qw( + 0000 + 5178 + 5218 + 5403 + 5424 + 5455 + 5485 + 5512 + 5545 + 5770 + 5797 + 5825 + 5854 + 5881 + 6138 + 6153 + 6197 + 6214 + ); + my @auxmask_end_date = qw( + 5177 + 5217 + 5402 + 5423 + 5454 + 5484 + 5511 + 5544 + 5769 + 5796 + 5824 + 5853 + 5880 + 6137 + 6152 + 6196 + 6213 + 9999 + ); + + my $outroot_base = basename($outroot); + + # Assume that outroot base begins with exp_name: fix by passing in exp_name + my $mjd = substr($outroot_base, 1, 4); + if ($mjd =~ /\D/) { + my $error_code = 1; + &my_die("unexpected outroot base: $outroot_base: $error_code", $chip_bg_id, $class_id, $error_code); + } + + my $i = 0; + while ($auxmask_end_date[$i] < $mjd) { + $i++; + } + + my $auxmask_base = "/data/ipp064.0/bills/m31-masks/www.usm.uni-muenchen.de/people/arri/PAndromeda/masks_fits/mask"; + + $auxmask = sprintf "%s_%04d-%04d_%s.fits", $auxmask_base, $auxmask_start_date[$i], $auxmask_end_date[$i], $class_id; + print "Auxililary mask file is $auxmask\n"; +} + + + # Run ppImage @@ -172,4 +247,6 @@ $command .= " -image $in_image"; $command .= " -mask $in_mask"; + $command .= " -variance $in_wt"; + $command .= " -auxmask $auxmask" if $auxmask; $command .= " -stats $out_stats" if $do_stats; $command .= " -background $in_bg" if $apply_bg; Index: trunk/ippScripts/scripts/background_warp.pl =================================================================== --- trunk/ippScripts/scripts/background_warp.pl (revision 35117) +++ trunk/ippScripts/scripts/background_warp.pl (revision 35118) @@ -29,4 +29,5 @@ my $missing_tools; my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); @@ -38,5 +39,5 @@ my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); -my ($astrometry, $imageName, $maskName, $magicked); +my ($astrometry, $imageName, $maskName, $weightName, $magicked); GetOptions( 'warp_bg_id|i=s' => \$warp_bg_id, # Warp identifier @@ -117,9 +118,32 @@ # Get list of filenames my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; + +# XXX: get this from recipe +my $alternate_astrometry_label = 'M31.test.20130206.bgsub'; +# XXX: get exp_id from command line +my $exp_id = 196758; + if (!defined $imageName) { # go find our inputs - my ($imageFile, $maskFile); + my $astrom_path_base; + if (defined $alternate_astrometry_label) { + my $command = "$camtool -processedexp -exp_id $exp_id -label $alternate_astrometry_label"; + $command .= " -dbname $dbname" if $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code); + } + my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR); + my $entry = $entries->[0]; + $astrom_path_base = $entry->{path_base}; + } + + my ($imageFile, $maskFile, $weightFile); ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); + ($weightFile, $weightName) = tempfile( "$tempOutRoot.wt.list.XXXX", UNLINK => !$save_temps); my $command = "$bgtool -warpinputs"; $command .= " -warp_bg_id $warp_bg_id"; @@ -139,19 +163,28 @@ my $image = $ipprc->filename("PPBACKGROUND.OUTPUT", $chip_path, $class_id); my $mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $chip_path, $class_id ); + my $wt = $ipprc->filename("PPBACKGROUND.OUTPUT.VARIANCE", $chip_path, $class_id ); print $imageFile "$image\n"; print $maskFile "$mask\n"; + print $weightFile "$wt\n"; &my_die("Can't find input image: $image", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($image); &my_die("Can't find input mask: $mask", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($mask); + &my_die("Can't find input mask: $wt", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($wt); &my_die("Magic status don't match: $magicked vs $file->{magicked}", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $magicked and $magicked != $file->{magicked}; $magicked = $file->{magicked}; - my $cam_path = $file->{cam_path_base}; - my $astrom = $ipprc->filename($astromSource, $cam_path); - &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; - $astrometry = $astrom; + if (!$astrom_path_base) { + my $cam_path = $file->{cam_path_base}; + my $astrom = $ipprc->filename($astromSource, $cam_path); + &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; + $astrometry = $astrom; + } elsif (!$astrometry) { + # using alternate astrometry + $astrometry = $ipprc->filename($astromSource, $astrom_path_base); + } } close $imageFile; close $maskFile; + close $weightFile; } @@ -166,4 +199,5 @@ my $out_image = prepare_output("PSWARP.OUTPUT", $outroot, $skycell_id, 1); my $out_mask = prepare_output("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1); +my $out_wt = prepare_output("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1); my $out_stats = prepare_output("SKYCELL.STATS", $outroot, $skycell_id, 1); my $out_config = prepare_output("PSWARP.CONFIG", $outroot, $skycell_id, 1); @@ -179,4 +213,5 @@ $command .= " -list $imageName"; $command .= " -masklist $maskName"; + $command .= " -variancelist $weightName"; $command .= " -astrom $astrometry"; $command .= " $outroot $skyFile"; @@ -220,4 +255,5 @@ &my_die("Couldn't find expected output file: $out_image", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); &my_die("Couldn't find expected output file: $out_mask", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); + &my_die("Couldn't find expected output file: $out_wt", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_wt); &my_die("Couldn't find expected output file: $out_config", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config); } Index: trunk/ppBackground/src/ppBackground.h =================================================================== --- trunk/ppBackground/src/ppBackground.h (revision 35117) +++ trunk/ppBackground/src/ppBackground.h (revision 35118) @@ -13,5 +13,8 @@ psString patternName; // Filename of pattern psString backgroundName; // Filename of background - psString imageName, maskName; // Filenames of input image/mask + psString imageName; // Filenames of input image + psString maskName; // Filenames of input mask + psString varianceName; // Filenames of input variance + psString auxMaskName; // Filename of auxillary mask psString outRoot; // Output root name psMetadata *stats; // Statistics for output @@ -52,5 +55,6 @@ const pmChip *pattern, // Chip with pattern const pmFPAview *view, // View to data - pmConfig *config // Configuration + pmConfig *config, // Configuration + psImageMaskType maskBad // value to use for bad pixels ); Index: trunk/ppBackground/src/ppBackgroundArguments.c =================================================================== --- trunk/ppBackground/src/ppBackgroundArguments.c (revision 35117) +++ trunk/ppBackground/src/ppBackgroundArguments.c (revision 35118) @@ -49,4 +49,6 @@ psMetadataAddStr(arguments, PS_LIST_TAIL, "-image", 0, "Filename of image (required)", NULL); psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask", 0, "Filename of mask", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-variance", 0, "Filename of variance image", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-auxmask", 0, "Filename of auxiliary mask", NULL); psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Output statistics file", NULL); psMetadataAddStr(arguments, PS_LIST_TAIL, "-dumpconfig", 0, "Output configuration file", NULL); @@ -59,4 +61,6 @@ data->imageName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-image")); data->maskName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-mask")); + data->varianceName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-variance")); + data->auxMaskName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-auxmask")); data->outRoot = psStringCopy(argv[1]); psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Output root name", data->outRoot); @@ -64,6 +68,6 @@ psTrace("ppBackground", 1, "Done reading command-line arguments\n"); - if (!data->patternName && !data->backgroundName) { - psError(PPBACKGROUND_ERR_CONFIG, true, "Must specify at least one of -pattern and -background"); + if (!data->patternName && !data->backgroundName && !data->auxMaskName) { + psError(PPBACKGROUND_ERR_CONFIG, true, "Must specify at least one of -pattern and -background -auxmask"); return false; } Index: trunk/ppBackground/src/ppBackgroundCamera.c =================================================================== --- trunk/ppBackground/src/ppBackgroundCamera.c (revision 35117) +++ trunk/ppBackground/src/ppBackgroundCamera.c (revision 35118) @@ -65,4 +65,20 @@ } + fileArguments("VARIANCE", data->varianceName, "Input uncorrected variance", data->config); + pmFPAfileBindFromArgs(&status, image, data->config, "PPBACKGROUND.VARIANCE", "VARIANCE"); // File + if (!status) { + psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.VARIANCE"); + return false; + } + + if (data->auxMaskName) { + fileArguments("AUXMASK", data->auxMaskName, "Auxiliary mask", data->config); + pmFPAfileBindFromArgs(&status, image, data->config, "PPBACKGROUND.AUXMASK", "AUXMASK"); // File + if (!status) { + psError(psErrorCodeLast(), false, "Failed to build file from PPBACKGROUND.MASK"); + return false; + } + } + pmFPAfile *output = pmFPAfileDefineOutput(data->config, image->fpa, "PPBACKGROUND.OUTPUT"); if (!output) { @@ -79,4 +95,11 @@ outMask->save = true; + pmFPAfile *outVariance = pmFPAfileDefineOutput(data->config, output->fpa, "PPBACKGROUND.OUTPUT.VARIANCE"); + if (!outVariance) { + psError(psErrorCodeLast(), false, "Unable to define output."); + return false; + } + outVariance->save = true; + #if 0 // Now the camera has been determined, we can read the recipe Index: trunk/ppBackground/src/ppBackgroundData.c =================================================================== --- trunk/ppBackground/src/ppBackgroundData.c (revision 35117) +++ trunk/ppBackground/src/ppBackgroundData.c (revision 35118) @@ -17,4 +17,6 @@ psFree(data->imageName); psFree(data->maskName); + psFree(data->varianceName); + psFree(data->auxMaskName); psFree(data->outRoot); psFree(data->stats); @@ -33,4 +35,6 @@ data->imageName = NULL; data->maskName = NULL; + data->varianceName = NULL; + data->auxMaskName = NULL; data->outRoot = NULL; data->stats = NULL; Index: trunk/ppBackground/src/ppBackgroundLoop.c =================================================================== --- trunk/ppBackground/src/ppBackgroundLoop.c (revision 35117) +++ trunk/ppBackground/src/ppBackgroundLoop.c (revision 35118) @@ -135,5 +135,5 @@ pmChip *patternChip = patternFile ? pmFPAviewThisChip(view, patternMosaic) : NULL; // Chip with pattern pmChip *bgChip = bgFile ? pmFPAviewThisChip(view, bgFile->fpa) : NULL; // Chip with background model - if (!ppBackgroundRestore(chip, bgChip, patternChip, view, config)) { + if (!ppBackgroundRestore(chip, bgChip, patternChip, view, config, maskBad)) { psError(psErrorCodeLast(), false, "Unable to replace background"); return false; Index: trunk/ppBackground/src/ppBackgroundRestore.c =================================================================== --- trunk/ppBackground/src/ppBackgroundRestore.c (revision 35117) +++ trunk/ppBackground/src/ppBackgroundRestore.c (revision 35118) @@ -35,5 +35,5 @@ bool ppBackgroundRestore(pmChip *chip, const pmChip *background, const pmChip *pattern, - const pmFPAview *oldView, pmConfig *config) + const pmFPAview *oldView, pmConfig *config, psImageMaskType maskBad) { PS_ASSERT_PTR_NON_NULL(chip, false); @@ -120,4 +120,23 @@ } + psImage *auxMask = psMetadataLookupPtr(NULL, ro->analysis, "EXPNUM"); + if (auxMask) { + if (auxMask->numCols != mask->numCols || auxMask->numRows != mask->numRows) { + psError(PPBACKGROUND_ERR_DATA, true, "auxiliary mask size (%d x %d) does not match input size (%d x %d)", + auxMask->numCols, auxMask->numRows, numCols, numRows); + return false; + } + // Arno's masks are floating point, fix later and zero means bad + #define AUXMASK_DATA F32 + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (auxMask->data.AUXMASK_DATA[y][x] == 0) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskBad; + } + } + } + } + + ro->data_exists = true; ro->parent->data_exists = true;