Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl (revision 35236) @@ -117,5 +117,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks", + -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks --dbname", -exitval => 3, ) unless @@ -124,5 +124,5 @@ defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode or - defined $confirm_stacks or defined $burntool_stats; + defined $confirm_stacks or defined $burntool_stats or defined $dbname; # Configurable parameters from our config file. @@ -282,7 +282,4 @@ unless(defined($camera)) { $camera = 'GPC1'; -} -unless(defined($dbname)) { - $dbname = 'gpc1'; } Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl (revision 35236) @@ -43,5 +43,5 @@ # Parse the command-line arguments my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, - $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked ); + $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $cam_path_base, $magicked ); GetOptions( 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier @@ -53,4 +53,5 @@ 'threads=s' => \$threads, # Number of threads to use 'chip_path_base=s' => \$chip_path_base, # optional chip_path_base + 'cam_path_base=s' => \$cam_path_base, # optional camera stage path_base 'magicked=s' => \$magicked, # magicked status of input 'verbose' => \$verbose, # Print to stdout @@ -94,4 +95,6 @@ my $dump_config; if ($chip_path_base and $no_update) { + # XXX: this path through the code has not been excercized in awhile and probably does not work + # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config # or do stats. Get path to input from command line. @@ -119,9 +122,15 @@ my $input = $$inputs[0]; # Input of interest $in_path = $input->{path_base}; + $cam_path_base = $input->{cam_path_base}; $magicked = $input->{magicked}; } my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id); -my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +my $in_mask; +if ($cam_path_base) { + $in_mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); +} else { + $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +} my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id); my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); @@ -177,4 +186,6 @@ my $apply_auxiliary_mask = 1; +# XXX Temporary lookup of auxiliary mask file from static location. +# At some point I'll convert this to register the masks in the database and use detselect if ($apply_auxiliary_mask) { my @auxmask_start_date = qw( @@ -238,5 +249,4 @@ print "Auxililary mask file is $auxmask\n"; } - @@ -272,5 +282,6 @@ &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); - my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP"; + my $resolved_stats = $ipprc->file_resolve($out_stats); + my $command = "$ppStatsFromMetadata $resolved_stats - BACKGROUND_CHIP"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl (revision 35236) @@ -119,27 +119,6 @@ my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; -# XXX: get this from recipe -my $alternate_astrometry_label = 'M31.test.20130206.bgsub'; -# XXX: get exp_id from command line -my $exp_id = 196758; - if (!defined $imageName) { # go find our inputs - my $astrom_path_base; - if (defined $alternate_astrometry_label) { - my $command = "$camtool -processedexp -exp_id $exp_id -label $alternate_astrometry_label"; - $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code); - } - my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR); - my $entry = $entries->[0]; - $astrom_path_base = $entry->{path_base}; - } - my ($imageFile, $maskFile, $weightFile); ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); @@ -174,13 +153,8 @@ $magicked = $file->{magicked}; - if (!$astrom_path_base) { - my $cam_path = $file->{cam_path_base}; - my $astrom = $ipprc->filename($astromSource, $cam_path); - &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; - $astrometry = $astrom; - } elsif (!$astrometry) { - # using alternate astrometry - $astrometry = $ipprc->filename($astromSource, $astrom_path_base); - } + my $cam_path = $file->{cam_path_base}; + my $astrom = $ipprc->filename($astromSource, $cam_path); + &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; + $astrometry = $astrom; } close $imageFile; @@ -238,5 +212,7 @@ &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); - my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP"; + my $real_stats = $ipprc->file_resolve($out_stats); + + my $command = "$ppStatsFromMetadata $real_stats - BACKGROUND_WARP"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl (revision 35236) @@ -379,4 +379,5 @@ check_output($bkgModel, $replicateOutputs); } + $cmdflags = $cmdflags . " -background_model 1 "; } } Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl (revision 35236) @@ -299,4 +299,9 @@ my $jpeg1Name = prepare_output("PPSUB.OUTPUT.JPEG1", $outroot, 1); my $jpeg2Name = prepare_output("PPSUB.OUTPUT.JPEG2", $outroot, 1); + +# do we always need to prepare this file? +if ($reduction ne 'NOCONVDIFF') { + my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 1); +} my ($inverseName, $inverseMask, $inverseVariance, $inverseSources); Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl (revision 35236) @@ -1957,4 +1957,5 @@ addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id); addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPBACKGROUND.OUTPUT.VARIANCE", $path_base, $class_id); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -2100,4 +2101,5 @@ addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id); addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename (\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl (revision 35236) @@ -92,4 +92,6 @@ my $comment; # Comment for exposure +my $exp_name_1; # Name of exposure 1 +my $exp_name_2; # Name of exposure 2 if ($stage eq 'camera') { my $command = "camtool -processedexp -cam_id $stage_id"; @@ -168,5 +170,5 @@ my %negative; # Data for negative diff detections foreach my $comp ( @$components ) { - my $path_base = $comp->{path_base}; # Base name for file + my $path_base = $comp->{path_base}; # Base name for file if (!$need_magic and $comp->{magicked}) { # This client is authorized to receive uncensored detections @@ -178,4 +180,6 @@ print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment}; $comment = $comp->{comment} unless defined $comment; + $exp_name_1 = $comp->{exp_name_1} unless defined $exp_name_1; + $exp_name_2 = $comp->{exp_name_2} unless defined $exp_name_2; (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time}; @@ -192,49 +196,49 @@ &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename); my $resolved = $ipprc->file_resolve($filename); - + &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved; $filename = $resolved; - my $cam_id = $comp->{cam_id_1}; - #print "Getting info from camera stage $cam_id\n"; - my $cam_command = "$camtool -processedexp -cam_id $cam_id"; - $cam_command .= " -dbname $dbname" if defined $dbname; - my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) = - run(command => $cam_command, verbose => $verbose); - &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success; - my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or - &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); - my $cam_components = parse_md_list($cam_metadata) or - &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); - &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1; - my $cam_comp = $$cam_components[0]; - - my $exp_id = $comp->{exp_id_1}; - #print "Getting info from raw stage $exp_id\n"; - my $reg_command = "$regtool -processedimfile -exp_id $exp_id"; - $reg_command .= " -dbname $dbname" if defined $dbname; - my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) = - run(command => $reg_command, verbose => $verbose); - &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success; - my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or - &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); - my $reg_components = parse_md_list($reg_metadata) or - &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); - my $reg_comp = $$reg_components[0]; - - # Now get the difftype value from diff->{diff_mode} - my $difftype = ""; - if ($comp->{diff_mode} == 1) { - $difftype = "WW"; - } elsif ($comp->{diff_mode} == 2) { - $difftype = "WS"; - } elsif ($comp->{diff_mode} == 3) { - $difftype = "SW"; # Not used yet - } elsif ($comp->{diff_mode} == 4) { - $difftype = "SS"; - } else { - $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]"; - } + my $cam_id = $comp->{cam_id_1}; + #print "Getting info from camera stage $cam_id\n"; + my $cam_command = "$camtool -processedexp -cam_id $cam_id"; + $cam_command .= " -dbname $dbname" if defined $dbname; + my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) = + run(command => $cam_command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success; + my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + my $cam_components = parse_md_list($cam_metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1; + my $cam_comp = $$cam_components[0]; + + my $exp_id = $comp->{exp_id_1}; + #print "Getting info from raw stage $exp_id\n"; + my $reg_command = "$regtool -processedimfile -exp_id $exp_id"; + $reg_command .= " -dbname $dbname" if defined $dbname; + my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) = + run(command => $reg_command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success; + my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + my $reg_components = parse_md_list($reg_metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + my $reg_comp = $$reg_components[0]; + + # Now get the difftype value from diff->{diff_mode} + my $difftype = ""; + if ($comp->{diff_mode} == 1) { + $difftype = "WW"; + } elsif ($comp->{diff_mode} == 2) { + $difftype = "WS"; + } elsif ($comp->{diff_mode} == 3) { + $difftype = "SW"; # Not used yet + } elsif ($comp->{diff_mode} == 4) { + $difftype = "SS"; + } else { + $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]"; + } my $data = { zp => $zp, @@ -250,12 +254,12 @@ output_format => $comp->{output_format}, direction => 1, - comment => $reg_comp->{comment}, - obsmode => $reg_comp->{obs_mode}, - difftype => $difftype, - sky => $cam_comp->{bg}, - shutoutc => $reg_comp->{dateobs}, + comment => $reg_comp->{comment}, + obsmode => $reg_comp->{obs_mode}, + difftype => $difftype, + sky => $cam_comp->{bg}, + shutoutc => $reg_comp->{dateobs}, }; - #warn("Checking for positive"); + #warn("Checking for positive"); diff_check(\%positive, $data, "positive"); @@ -288,12 +292,12 @@ diff_id => $comp->{diff_id}, direction => 0, - comment => $reg_comp->{comment}, - obsmode => $reg_comp->{obs_mode}, - difftype => $difftype, - sky => $cam_comp->{bg}, - shutoutc => $reg_comp->{dateobs}, + comment => $reg_comp->{comment}, + obsmode => $reg_comp->{obs_mode}, + difftype => $difftype, + sky => $cam_comp->{bg}, + shutoutc => $reg_comp->{dateobs}, }; - #warn("Checking for negative"); + #warn("Checking for negative"); diff_check(\%negative, $data, "negative"); @@ -326,4 +330,6 @@ my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $dsType --list $dsFileName"; $command .= " --ps0 \"$comment\"" if defined $comment; + $command .= " --ps1 \"$exp_name_1\"" if defined $exp_name_1; + $command .= " --ps2 \"$exp_name_2\"" if defined $exp_name_2; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl (revision 35236) @@ -231,5 +231,6 @@ $command .= " -F PSASTRO.OUTPUT $output_sources_filerule"; $command .= " -stats $stats"; - $command .= " -recipe PPSTATS CAMSTATS"; + # $command .= " -recipe PPSTATS CAMSTATS"; + $command .= " -recipe PPSTATS SKYCALSTATS"; $command .= " -dumpconfig $configuration" if $configuration; $command .= " -tracedest $traceDest"; Index: branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl (revision 35125) +++ branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl (revision 35236) @@ -245,6 +245,12 @@ print $weightFile "$weight\n"; my $bkg; - if ($doBackground) { - $bkg = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id}); + if (($doBackground)) { + # &&($imfile->{background_model} == 1) + if ($imfile->{cam_background_model}) { + $bkg = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id}); + } + else { + $bkg = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id}); + } print $bkgFile "$bkg\n"; }