Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/mergedvodb_merge.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/mergedvodb_merge.pl (revision 38054) @@ -156,5 +156,6 @@ $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; } else { - $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path"; +# $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path"; + $merge_command = "$dvomerge -matched-tables $minidvodb_path into $mergedvodb_path"; } print "\n$merge_command\n"; Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/psphot_fullforce_warp.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/psphot_fullforce_warp.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/psphot_fullforce_warp.pl (revision 38054) @@ -124,8 +124,5 @@ } -# XXX: need to figure out whether this works or not -# We probably want to create a specific recipe that looks at the results -my $recipe_ppstats = 'WARPSTATS'; -my $doStats = 0; +my $doStats = 1; print "reduction: $reduction\n"; @@ -166,4 +163,5 @@ my $cmdflags = ""; +my $quality = 0; # Perform psphotFullForce @@ -182,5 +180,4 @@ if ($doStats) { $command .= " -stats $outputStats"; - $command .= " -recipe PPSTATS $recipe_ppstats"; } $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; @@ -198,10 +195,9 @@ } - # Stats: TODO if ($doStats) { check_output($outputStats, 1); my $outputStatsReal = $ipprc->file_resolve($outputStats); - # measure chip stats + # analyze the stats file $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -216,12 +212,13 @@ chomp $cmdflags; } - my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - - if (!$quality) { - check_output($outputSources, 1); - } + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + } else { print "Not executing: $command\n"; } +} + +if (!$quality) { + check_output($outputSources, 1); } Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_camera.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_camera.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_camera.pl (revision 38054) @@ -14,4 +14,5 @@ use Data::Dumper; use File::Basename; +use File::Temp qw( tempfile ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -30,4 +31,5 @@ my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1); my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); if ($missing_tools) { @@ -88,4 +90,5 @@ my ($workdir,$exp_tag,$reduction); { +### $command = "$camtool -processedexp -cam_id $cam_id "; ### THIS LINE ONLY TO TEST! my $command = "$camtool -pendingexp -cam_id $cam_id "; $command .= " -dbname $dbname " if defined($dbname); @@ -142,4 +145,30 @@ my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist"; my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist"; + + # Despite the previous comment, we do care about the output of pendingimfile, as that + # contains information for the exposure. + my ($statFile, $statName) = tempfile( "/tmp/cm.$cam_id.stats.XXXX", UNLINK => 1 ); + foreach my $line (@$stdout_buf) { + print $statFile $line; + } + close $statFile; + + # parse the stats in the metadata file + $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform ppStatsFromMetadata: $error_code\n"); + exit($error_code); + } + + my $results = ''; + foreach my $line (@$stdout_buf) { + $results .= " $line"; + } + chomp($results); + + my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use @@ -163,4 +192,5 @@ $camtool_post_cmd .= " -dbname $dbname " if defined $dbname; $camtool_post_cmd .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 "; + $camtool_post_cmd .= " $results "; # Add exposure information my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo "; my $post_cmd_SfM = " ppStatsFromMetadata ${remote_outroot}.stats - CAMERA_EXP_FPA >> ${remote_outroot}.dbinfo "; Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_stack.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_stack.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/sc_prepare_stack.pl (revision 38054) @@ -23,4 +23,5 @@ my $remote_hostname = "LANL/Mustang"; # Name of the remote node. my $threads_req = 4; # How many threads are we going to use? +my $have_warps = 0; # Do we have a copy of the warp if we don't find one there? my $fail_state = "prep_fail"; @@ -156,19 +157,44 @@ my $ipp_disk; # ${remote_root}/tmp/${ipp_disk} - (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits"); - ($ipp_disk,undef) = &uri_convert($warp_path_base . ".fits"); - print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; - (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits"); - ($ipp_disk,undef) = &uri_convert($warp_path_base . ".wt.fits"); - print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; - (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits"); - ($ipp_disk,undef) = &uri_convert($warp_path_base . ".mask.fits"); - print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; - (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf"); - ($ipp_disk,undef) = &uri_convert($warp_path_base . ".cmf"); - print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; - (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf"); - ($ipp_disk,undef) = &uri_convert($warp_path_base . ".psf"); - print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + if ($have_warps != 0) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing. + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".wt.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".mask.fits"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".cmf"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf"); + ($ipp_disk,undef) = &uri_convert($warp_path_base . ".psf"); + print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n"; + + } + else { + $remote_file = &uri_local_to_remote($warp_path_base . ".fits"); + print COMPMAP "$remote_file $remote_file\n"; # No other option, really. + + $remote_file = &uri_local_to_remote($warp_path_base . ".wt.fits"); + print COMPMAP "$remote_file $remote_file\n"; # No other option, really. + + $remote_file = &uri_local_to_remote($warp_path_base . ".mask.fits"); + print COMPMAP "$remote_file $remote_file\n"; # No other option, really. + + $remote_file = &uri_local_to_remote($warp_path_base . ".cmf"); + print COMPMAP "$remote_file $remote_file\n"; # No other option, really. + + $remote_file = &uri_local_to_remote($warp_path_base . ".psf"); + print COMPMAP "$remote_file $remote_file\n"; # No other option, really. + + } } Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/skycalibration.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/skycalibration.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/skycalibration.pl (revision 38054) @@ -235,4 +235,5 @@ $command .= " -dumpconfig $configuration" if $configuration; $command .= " -tracedest $traceDest"; + $command .= " -dbname $dbname" if defined $dbname; unless ($no_op) { Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/stack_bkg_mk_mdc.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 38054) @@ -12,5 +12,5 @@ -my ($dbname,$stack_id,$label,$camera,$filter); +my ($dbname,$stack_id,$sass_id,$label,$camera,$filter); my $ipprc = PS::IPP::Config->new(); # IPP Configuration @@ -21,4 +21,5 @@ GetOptions( 'stack_id=s' => \$stack_id, + 'sass_id=s' => \$sass_id, 'label=s' => \$label, 'filter=s' => \$filter, @@ -26,5 +27,5 @@ 'camera=s' => \$camera, ) or pod2usage ( 2 ); -pod2usage( -msg => "Required options; --stack_id | --label", -exitval => 3) unless +pod2usage( -msg => "Required options; --stack_id | --label | --sass_id", -exitval => 3) unless ((defined $stack_id)||(defined $label)); pod2usage( -msg => "Required options; --camera --dbname ", -exitval => 3) unless @@ -49,9 +50,10 @@ -my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 "; +my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN stackAssociationMap USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 "; if (defined($label)) { $query .= " AND stackRun.label = '${label}' "; } if (defined($stack_id)) { $query .= " AND stack_id = ${stack_id} "; } if (defined($filter)) { $query .= " AND stackRun.filter = '${filter}' "; } +if (defined($sass_id)) { $query .= " AND sass_id = ${sass_id} "; } my $cams = $db->selectall_arrayref( $query ); Index: /branches/eam_branches/ipp-20150112/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/eam_branches/ipp-20150112/ippScripts/scripts/staticsky.pl (revision 38053) +++ /branches/eam_branches/ipp-20150112/ippScripts/scripts/staticsky.pl (revision 38054) @@ -48,4 +48,5 @@ print " --outroot (root) : Output root name\n"; print " --reduction (class) : Reduction class\n"; + print " --require_sources : Require that the stack source files exist\n"; print " --verbose : Be extra verbose\n"; print " --no-update : Don't update the database?\n"; @@ -60,5 +61,5 @@ print "run staticsky.pl @ARGV\n"; -my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $require_sources, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( 'sky_id=s' => \$sky_id, # Diff identifier @@ -68,4 +69,5 @@ 'outroot=s' => \$outroot, # Output root name 'reduction=s' => \$reduction, # Reduction class + 'require_sources' => \$require_sources, # Require that the stack sources exist. 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Don't update the database? @@ -178,6 +180,10 @@ &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); } - &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); - + + my $have_sources = $ipprc->file_exists("$sources"); + if ($require_sources) { + &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $have_sources; + } + print $listFile " STACK_ID S64 " . $stack_id . "\n"; print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; @@ -185,5 +191,7 @@ print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; print $listFile " RAW:EXPNUM STR " . $expnumRaw . "\n"; - print $listFile " SOURCES STR " . $sources . "\n"; + if ($have_sources) { + print $listFile " SOURCES STR " . $sources . "\n"; + } if ($needConvolvedImages) { print $listFile " CNV:IMAGE STR " . $imageCnv . "\n";