Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl (revision 31587) @@ -153,4 +153,28 @@ } chomp $cmdflags; + + { # Determine if FWHM is too large to bother continuing. + my $command = "$ppConfigDump -camera $camera -dump-camera -"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR); + } + my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR); + + my $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); + if ($maxFWHM) { + my $expFWHM; + ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/; + + if ($expFWHM > $maxFWHM) { + print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM maximum: $maxFWHM\n"; + $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA + $no_op = 1; + } + } + } } Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl (revision 31587) @@ -114,5 +114,5 @@ $tool_cmd = "$staticskytool -sky_id"; $list_mode = "-result"; - $component_key = ""; + $component_key = "skycell_id"; } elsif ($stage eq "diff") { $tool_cmd = "$difftool -diff_id"; Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl (revision 31587) @@ -62,4 +62,5 @@ my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); my $file_cmd = can_run('file') or (warn "can't find program file" and $missing_tools = 1); my $zcat = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1); @@ -106,25 +107,30 @@ defined $stage_id and defined $component and - defined $exp_type and defined $path_base and defined $outroot; +if ($stage eq 'raw' and !$clean and !$no_magic) { + + # for raw stage need to have exposure type defined and if the type is OBJECT we need + # a chip_path_base so we can find the chip mask file + if (!defined $exp_type or ($exp_type eq 'OBJECT' and !defined $chip_path_base)) { + pod2usage( -msg => "Required options: --chip_path_base --exp_type for raw stage", -exitval => 3); + } +} + $ipprc->redirect_output($logfile) if $logfile; -if ($stage eq 'raw' and !$clean and !$no_magic) { - # need to be able to find chip mask file - if ($exp_type eq 'OBJECT' and !defined $chip_path_base) { - pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3); - } -} - $ipprc->define_camera($camera); $ipprc->outroot_prepare($outroot); +my $num_sky_inputs; +if ($stage eq 'sky') { + $num_sky_inputs = get_num_sky_inputs($stage_id); +} # Get the list of data products for this component # note: We my_die in get_file_list if something goes wrong. -my $file_list = get_file_list($stage, $component, $path_base, $clean); +my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs); if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) { @@ -184,4 +190,8 @@ # it must exist) next if $poor_quality; + + # skip file stats file. Due to a bug the update process destroys them sometimes + # XXX: perhaps only do this for stages where we know that this happens + next if $file_rule =~ /STATS/; &my_die("failed to resolve $file_name", $component, $PS_EXIT_DATA_ERROR); @@ -480,4 +490,5 @@ my $path_base = shift; my $clean = shift; + my $num_sky_inputs = shift; my @file_list; @@ -531,5 +542,9 @@ $config_file_rule = "PPSTACK.CONFIG"; } elsif ($stage eq "sky") { - $config_file_rule = "PSPHOT.STACK.CONFIG"; + if ($num_sky_inputs > 1) { + $config_file_rule = "PSPHOT.STACK.CONFIG"; + } else { + $config_file_rule = "PSPHOT.SKY.CONFIG"; + } } else { &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR); @@ -627,4 +642,24 @@ return \@file_list; } +sub get_num_sky_inputs { + my $sky_id = shift; + + my $command = "$staticskytool -inputs -sky_id $sky_id -simple"; + $command .= " -dbname $dbname" if $dbname; + $command .= " | wc"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $component, $error_code); + } + my ($num_inputs, $words, $chars) = split " ", (join "", @$stdout_buf); + if (!$num_inputs) { + $num_inputs = "undefined" if !defined $num_inputs; + &my_die("unexpected number of static sky inputs $num_inputs", $PS_EXIT_PROG_ERROR, $component, $error_code); + } + + return $num_inputs; +} sub my_die Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl (revision 31587) @@ -99,5 +99,5 @@ my $status = queue_chips($lap_id); - if ($status) { + if (!$status) { # This is the culprit. my $command = "$laptool -updaterun -lap_id $lap_id"; $command .= " -dbname $dbname " if defined $dbname; @@ -120,5 +120,5 @@ # This is a puzzler... chiptool doesn't actually return a useful metadata. We'll just scrape it from the database for now. - my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group"; + my $command = "$chiptool -listrun -pstamp_order -exp_id $exp_id -label $label -data_group $data_group"; $command .= " -dbname $dbname " if defined $dbname; @@ -236,4 +236,29 @@ my $comment = $exposure->{comment}; + # This is a hack to fix old exposures that have no object. + unless(defined($comment)) { + $comment = ''; + } + if ((!defined($object))||($object eq 'NULL')||($object eq '')) { + if ($comment =~ /3pi_/) { + $object = $comment; + $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/; + } + elsif ($comment =~ / ps1_/) { + $object = $comment; + $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/; + } + else { + $exposure->{data_state} = 'new'; + $exposure->{pairwise} = 0; + $exposure->{private} = 1; + $exposure->{pair_id} = 9223372036854775807; + update_this_exposure($exposure); + $counter++; + next; + } + } + + if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id if (($pairwise) && !($pair_id)) { @@ -255,4 +280,5 @@ $counter++; } + print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; } @@ -262,4 +288,5 @@ foreach my $comment (keys %{ $matching{$object} }) { push @exp_ids_to_diff, $matching{$object}{$comment}; + print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n"; } @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; @@ -275,5 +302,5 @@ my $exp_A = ${ $exposures }[$indexing{$exp_id_A}]; my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; - + print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n"; $exp_A->{pairwise} = 1; $exp_A->{private} = 0; @@ -292,4 +319,5 @@ # Scan all exposures, and ensure that pairwise and private are set correctly foreach my $exposure (@$exposures) { + print "YY: $exposure\n"; if ($exposure->{pairwise} && !($exposure->{pair_id})) { $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. @@ -299,7 +327,7 @@ } - $exposure = update_this_exposure($exposure); - } - return(1); + update_this_exposure($exposure); + } + return(0); } @@ -432,7 +460,13 @@ my $companion; - if ($exposure->{pair_id}) { # Load companion exposure information - if (exists($match_hash{$exposure->{chip_id}})) { + if ($exposure->{pairwise}) { + # Load companion exposure information + if (($exposure->{pair_id})&&(exists($match_hash{$exposure->{chip_id}}))) { $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match! + } + else { # We claimed to be pairwise, but do not have a valid pair_id. + $exposure->{pairwise} = 0; + $exposure->{private} = 1; + &update_this_exposure($exposure); } } @@ -446,26 +480,49 @@ $exposure = remake_this_exposure($exposure); } - if ($exposure->{cam_quality}) { - $needs_qstack = 1; - $needs_something_private = 1; - if ($companion) { - $companion->{private} = 1; - $companion->{pairwise} = 0; - &update_this_exposure($companion); + # Do quality checks here + my $is_bad_quality = 0; + if ((defined($exposure->{chipRun_state}))&&($exposure->{chipRun_state} eq 'full')&& + ($exposure->{chip_component_count} > 0)&&($exposure->{chip_bad_quality} / $exposure->{chip_component_count} > 0.05)) { + printf("QUALITY: $exposure->{exp_id} has bad chip quality: %d / %d\n", + $exposure->{chip_bad_quality} , $exposure->{chip_component_count}); + $is_bad_quality = 1; + } + elsif ((defined($exposure->{camRun_state}))&&($exposure->{camRun_state} eq 'full')&& + ($exposure->{cam_bad_quality} / $exposure->{cam_component_count} > 0)) { + printf("QUALITY: $exposure->{exp_id} has bad cam quality: %d / %d\n", + $exposure->{cam_bad_quality} , $exposure->{cam_component_count}); + $is_bad_quality = 1; + } + elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&& + ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) { + printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n", + $exposure->{warp_bad_quality} , $exposure->{warp_component_count}); + $is_bad_quality = 1; + } + elsif ((defined($exposure->{diffRun_state}))&&($exposure->{diffRun_state} eq 'full')&& + ($exposure->{diff_bad_quality} / $exposure->{diff_component_count} > 0.5)) { + printf("QUALITY: $exposure->{exp_id} has bad diff quality: %d / %d\n", + $exposure->{diff_bad_quality} , $exposure->{diff_component_count}); + $is_bad_quality = 1; + } + # If we've detected a bad quality exposure, drop it, and tell the companion. + if ($is_bad_quality) { + unless ((defined($exposure->{diffRun_state}))&& + ($exposure->{diffRun_state} eq 'full')) { + $needs_qstack = 1; + $needs_something_private = 1; + if ($companion) { + $companion->{private} = 1; + $companion->{pairwise} = 0; + &update_this_exposure($companion); + } + $exposure->{private} = 1; + $exposure->{pairwise} = 0; } - $exposure->{private} = 1; - $exposure->{pairwise} = 0; $exposure->{data_state} = 'drop'; &update_this_exposure($exposure); } -# if ($companion) { # Validate that there are no problems with the companion exposure -# if ($companion->{cam_quality}) { # Maybe other things here? -# $exposure->{private} = 1; -# $exposure->{data_state} = 'drop'; -# &update_this_exposure($exposure); -# $needs_qstack = 1; -# } -# } + if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. $can_qstack ++; @@ -564,4 +621,12 @@ } + my $warps = ''; + foreach $exposure (@$exposures) { + if (($exposure->{data_state} != 'drop')&& + (S64_IS_NOT_NULL($exposure->{warp_id}))) { + $warps .= " -warp_id $exposure->{warp_id} "; + } + } + my @utctime = gmtime(); $utctime[5] += 1900; @@ -570,5 +635,5 @@ my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; - my $data_group = "${label}.${date}"; + my $data_group = "${label}.${proj_cell}.quick.${date}"; my $command = "$stacktool "; @@ -576,7 +641,8 @@ $command .= " -dbname $dbname " if defined $dbname; $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; - $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} "; + $command .= " -set_label ${label} -set_data_group $data_group "; $command .= " -set_workdir $workdir -set_dist_group NODIST "; $command .= " -min_num 2 -set_reduction QUICKSTACK "; + $command .= " $warps "; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -589,5 +655,5 @@ $command = "$stacktool "; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} "; + $command .= " -sassskyfile -data_group $data_group "; $command .= " -filter $filter -projection_cell ${proj_cell} "; @@ -632,4 +698,13 @@ unless (defined($label) && defined($filter) && defined($proj_cell)) { &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my $warps = ''; + foreach $exposure (@$exposures) { + if (($exposure->{data_state} != 'drop')&& + (S64_IS_NOT_NULL($exposure->{magicked}))&& + (S64_IS_NOT_NULL($exposure->{warp_id}))) { + $warps .= " -warp_id $exposure->{warp_id} "; + } } @@ -640,5 +715,5 @@ my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; - my $data_group = "${label}.${date}"; + my $data_group = "${label}.${proj_cell}.final.${date}"; my $command = "$stacktool "; @@ -646,6 +721,7 @@ $command .= " -dbname $dbname " if defined $dbname; $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; - $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} "; - $command .= " -min_num 2 -set_reduction THREEPI_STACK "; + $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; + $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group ${label} "; + $command .= " $warps "; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -659,5 +735,5 @@ $command = "$stacktool "; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} "; + $command .= " -sassskyfile -data_group $data_group "; $command .= " -filter $filter -projection_cell ${proj_cell} "; @@ -734,10 +810,12 @@ $command .= " -set_dist_group $exposure->{dist_group} "; } - - if ($exposure->{pairwise}) { # warpwarp + my $retry_command; + if (($exposure->{pairwise})&&(defined(${ $exposures }[$match_hash{$exposure->{chip_id}}]))) { # warpwarp my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; $command .= " -definewarpwarp "; - $command .= "-input_label $label -template_label $label -backwards "; + $command .= "-input_label $label -template_label $label "; $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; + $retry_command = $command; + $command .= " -backwards "; # This usually works. $already_queued{$exposure->{warp_id}} = 1; $already_queued{$companion->{warp_id}} = 1; @@ -763,8 +841,23 @@ my $diff_id = $diff->{diff_id}; unless (defined($diff_id)) { - $exposure->{data_state} = 'drop'; - &update_this_exposure($exposure); - } - + if ($retry_command) { + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $retry_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } + + $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, ""); + + $diff = ${ $diffs }[0]; + $diff_id = $diff->{diff_id}; + } + unless (defined($diff_id)) { + $exposure->{data_state} = 'drop'; + &update_this_exposure($exposure); + } + } } } @@ -872,4 +965,5 @@ my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id "; + $command .= " -dbname $dbname " if defined $dbname; if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) { $command .= " -set_chip_id $exposure->{chip_id} "; Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl (revision 31587) @@ -224,6 +224,6 @@ $stage = 'sky'; # XXX: This should be skycell, but the distribution code uses exposure - $comp_name = 'exposure'; - $current_component = $comp_name; + $comp_name = 'skycell_id'; +# $current_component = $comp_name; } else { &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); @@ -237,4 +237,13 @@ $new_workdir_value = "$workdir/$destdir"; } + + if ($stage eq 'sky') { + # the dbinfo file for a skyRun only has one component and it doesn't contain + # skycell_id which is the way components are listed in the dirinfo file. + my @ids = keys %$components; + &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1; + $current_component = $ids[0]; + } + my $component_dir; if ($current_component) { Index: branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl (revision 31458) +++ branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl (revision 31587) @@ -72,7 +72,9 @@ my ($logDest, $traceDest); +my $do_stats; if ($run_state eq 'new') { $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0); $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); + $do_stats = 1; } elsif ($run_state eq 'update') { $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1); @@ -162,5 +164,8 @@ my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1); my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1); -my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1); +my $outputStats; +if ($do_stats) { + $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats; +} my $configuration; @@ -257,5 +262,4 @@ # Run pswarp my $cmdflags; -my $do_stats; unless ($no_op) { my $command = "$pswarp"; @@ -279,5 +283,4 @@ if ($run_state eq 'new') { $command .= " -dumpconfig $configuration"; - $do_stats = 1; } else { $command .= " -ipprc $configuration";