- Timestamp:
- May 29, 2011, 11:26:31 AM (15 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl
r31454 r31587 99 99 my $status = queue_chips($lap_id); 100 100 101 if ( $status) {101 if (!$status) { # This is the culprit. 102 102 my $command = "$laptool -updaterun -lap_id $lap_id"; 103 103 $command .= " -dbname $dbname " if defined $dbname; … … 120 120 # This is a puzzler... chiptool doesn't actually return a useful metadata. We'll just scrape it from the database for now. 121 121 122 my $command = "$chiptool -listrun - exp_id $exp_id -label $label -data_group $data_group";122 my $command = "$chiptool -listrun -pstamp_order -exp_id $exp_id -label $label -data_group $data_group"; 123 123 $command .= " -dbname $dbname " if defined $dbname; 124 124 … … 236 236 my $comment = $exposure->{comment}; 237 237 238 # This is a hack to fix old exposures that have no object. 239 unless(defined($comment)) { 240 $comment = ''; 241 } 242 if ((!defined($object))||($object eq 'NULL')||($object eq '')) { 243 if ($comment =~ /3pi_/) { 244 $object = $comment; 245 $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/; 246 } 247 elsif ($comment =~ / ps1_/) { 248 $object = $comment; 249 $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/; 250 } 251 else { 252 $exposure->{data_state} = 'new'; 253 $exposure->{pairwise} = 0; 254 $exposure->{private} = 1; 255 $exposure->{pair_id} = 9223372036854775807; 256 update_this_exposure($exposure); 257 $counter++; 258 next; 259 } 260 } 261 262 238 263 if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id 239 264 if (($pairwise) && !($pair_id)) { … … 255 280 $counter++; 256 281 } 282 print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; 257 283 } 258 284 … … 262 288 foreach my $comment (keys %{ $matching{$object} }) { 263 289 push @exp_ids_to_diff, $matching{$object}{$comment}; 290 print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n"; 264 291 } 265 292 @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; … … 275 302 my $exp_A = ${ $exposures }[$indexing{$exp_id_A}]; 276 303 my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; 277 304 print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n"; 278 305 $exp_A->{pairwise} = 1; 279 306 $exp_A->{private} = 0; … … 292 319 # Scan all exposures, and ensure that pairwise and private are set correctly 293 320 foreach my $exposure (@$exposures) { 321 print "YY: $exposure\n"; 294 322 if ($exposure->{pairwise} && !($exposure->{pair_id})) { 295 323 $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. … … 299 327 } 300 328 301 $exposure =update_this_exposure($exposure);302 } 303 return( 1);329 update_this_exposure($exposure); 330 } 331 return(0); 304 332 } 305 333 … … 432 460 my $companion; 433 461 434 if ($exposure->{pair_id}) { # Load companion exposure information 435 if (exists($match_hash{$exposure->{chip_id}})) { 462 if ($exposure->{pairwise}) { 463 # Load companion exposure information 464 if (($exposure->{pair_id})&&(exists($match_hash{$exposure->{chip_id}}))) { 436 465 $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match! 466 } 467 else { # We claimed to be pairwise, but do not have a valid pair_id. 468 $exposure->{pairwise} = 0; 469 $exposure->{private} = 1; 470 &update_this_exposure($exposure); 437 471 } 438 472 } … … 446 480 $exposure = remake_this_exposure($exposure); 447 481 } 448 if ($exposure->{cam_quality}) { 449 $needs_qstack = 1; 450 $needs_something_private = 1; 451 if ($companion) { 452 $companion->{private} = 1; 453 $companion->{pairwise} = 0; 454 &update_this_exposure($companion); 482 # Do quality checks here 483 my $is_bad_quality = 0; 484 if ((defined($exposure->{chipRun_state}))&&($exposure->{chipRun_state} eq 'full')&& 485 ($exposure->{chip_component_count} > 0)&&($exposure->{chip_bad_quality} / $exposure->{chip_component_count} > 0.05)) { 486 printf("QUALITY: $exposure->{exp_id} has bad chip quality: %d / %d\n", 487 $exposure->{chip_bad_quality} , $exposure->{chip_component_count}); 488 $is_bad_quality = 1; 489 } 490 elsif ((defined($exposure->{camRun_state}))&&($exposure->{camRun_state} eq 'full')&& 491 ($exposure->{cam_bad_quality} / $exposure->{cam_component_count} > 0)) { 492 printf("QUALITY: $exposure->{exp_id} has bad cam quality: %d / %d\n", 493 $exposure->{cam_bad_quality} , $exposure->{cam_component_count}); 494 $is_bad_quality = 1; 495 } 496 elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&& 497 ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) { 498 printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n", 499 $exposure->{warp_bad_quality} , $exposure->{warp_component_count}); 500 $is_bad_quality = 1; 501 } 502 elsif ((defined($exposure->{diffRun_state}))&&($exposure->{diffRun_state} eq 'full')&& 503 ($exposure->{diff_bad_quality} / $exposure->{diff_component_count} > 0.5)) { 504 printf("QUALITY: $exposure->{exp_id} has bad diff quality: %d / %d\n", 505 $exposure->{diff_bad_quality} , $exposure->{diff_component_count}); 506 $is_bad_quality = 1; 507 } 508 # If we've detected a bad quality exposure, drop it, and tell the companion. 509 if ($is_bad_quality) { 510 unless ((defined($exposure->{diffRun_state}))&& 511 ($exposure->{diffRun_state} eq 'full')) { 512 $needs_qstack = 1; 513 $needs_something_private = 1; 514 if ($companion) { 515 $companion->{private} = 1; 516 $companion->{pairwise} = 0; 517 &update_this_exposure($companion); 518 } 519 $exposure->{private} = 1; 520 $exposure->{pairwise} = 0; 455 521 } 456 $exposure->{private} = 1;457 $exposure->{pairwise} = 0;458 522 $exposure->{data_state} = 'drop'; 459 523 &update_this_exposure($exposure); 460 524 461 525 } 462 # if ($companion) { # Validate that there are no problems with the companion exposure 463 # if ($companion->{cam_quality}) { # Maybe other things here? 464 # $exposure->{private} = 1; 465 # $exposure->{data_state} = 'drop'; 466 # &update_this_exposure($exposure); 467 # $needs_qstack = 1; 468 # } 469 # } 526 470 527 if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. 471 528 $can_qstack ++; … … 564 621 } 565 622 623 my $warps = ''; 624 foreach $exposure (@$exposures) { 625 if (($exposure->{data_state} != 'drop')&& 626 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 627 $warps .= " -warp_id $exposure->{warp_id} "; 628 } 629 } 630 566 631 my @utctime = gmtime(); 567 632 $utctime[5] += 1900; … … 570 635 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 571 636 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 572 my $data_group = "${label}.${ date}";637 my $data_group = "${label}.${proj_cell}.quick.${date}"; 573 638 574 639 my $command = "$stacktool "; … … 576 641 $command .= " -dbname $dbname " if defined $dbname; 577 642 $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; 578 $command .= " -set_label ${label} -set_data_group $ {proj_cell}.quick.${date}";643 $command .= " -set_label ${label} -set_data_group $data_group "; 579 644 $command .= " -set_workdir $workdir -set_dist_group NODIST "; 580 645 $command .= " -min_num 2 -set_reduction QUICKSTACK "; 646 $command .= " $warps "; 581 647 582 648 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 589 655 $command = "$stacktool "; 590 656 $command .= " -dbname $dbname " if defined $dbname; 591 $command .= " -sassskyfile -data_group $ {proj_cell}.quick.${date}";657 $command .= " -sassskyfile -data_group $data_group "; 592 658 $command .= " -filter $filter -projection_cell ${proj_cell} "; 593 659 … … 632 698 unless (defined($label) && defined($filter) && defined($proj_cell)) { 633 699 &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); 700 } 701 702 my $warps = ''; 703 foreach $exposure (@$exposures) { 704 if (($exposure->{data_state} != 'drop')&& 705 (S64_IS_NOT_NULL($exposure->{magicked}))&& 706 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 707 $warps .= " -warp_id $exposure->{warp_id} "; 708 } 634 709 } 635 710 … … 640 715 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 641 716 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 642 my $data_group = "${label}.${ date}";717 my $data_group = "${label}.${proj_cell}.final.${date}"; 643 718 644 719 my $command = "$stacktool "; … … 646 721 $command .= " -dbname $dbname " if defined $dbname; 647 722 $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; 648 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} "; 649 $command .= " -min_num 2 -set_reduction THREEPI_STACK "; 723 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; 724 $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group ${label} "; 725 $command .= " $warps "; 650 726 651 727 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = … … 659 735 $command = "$stacktool "; 660 736 $command .= " -dbname $dbname " if defined $dbname; 661 $command .= " -sassskyfile -data_group $ {proj_cell}.final.${date}";737 $command .= " -sassskyfile -data_group $data_group "; 662 738 $command .= " -filter $filter -projection_cell ${proj_cell} "; 663 739 … … 734 810 $command .= " -set_dist_group $exposure->{dist_group} "; 735 811 } 736 737 if ( $exposure->{pairwise}) { # warpwarp812 my $retry_command; 813 if (($exposure->{pairwise})&&(defined(${ $exposures }[$match_hash{$exposure->{chip_id}}]))) { # warpwarp 738 814 my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; 739 815 $command .= " -definewarpwarp "; 740 $command .= "-input_label $label -template_label $label -backwards";816 $command .= "-input_label $label -template_label $label "; 741 817 $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; 818 $retry_command = $command; 819 $command .= " -backwards "; # This usually works. 742 820 $already_queued{$exposure->{warp_id}} = 1; 743 821 $already_queued{$companion->{warp_id}} = 1; … … 763 841 my $diff_id = $diff->{diff_id}; 764 842 unless (defined($diff_id)) { 765 $exposure->{data_state} = 'drop'; 766 &update_this_exposure($exposure); 767 } 768 843 if ($retry_command) { 844 ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 845 run(command => $retry_command, verbose => $verbose); 846 unless ($success) { 847 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 848 &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); 849 } 850 851 $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or 852 &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, ""); 853 854 $diff = ${ $diffs }[0]; 855 $diff_id = $diff->{diff_id}; 856 } 857 unless (defined($diff_id)) { 858 $exposure->{data_state} = 'drop'; 859 &update_this_exposure($exposure); 860 } 861 } 769 862 } 770 863 } … … 872 965 873 966 my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id "; 967 $command .= " -dbname $dbname " if defined $dbname; 874 968 if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) { 875 969 $command .= " -set_chip_id $exposure->{chip_id} ";
Note:
See TracChangeset
for help on using the changeset viewer.
