Index: /tags/ipp-ps1-20210708/ippScripts/scripts/nightly_science.pl =================================================================== --- /tags/ipp-ps1-20210708/ippScripts/scripts/nightly_science.pl (revision 42298) +++ /tags/ipp-ps1-20210708/ippScripts/scripts/nightly_science.pl (revision 42299) @@ -142,4 +142,7 @@ my %unique_filter_hash = (); +#set up the camera config +my $ipprcam = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up camera", $PS_EXIT_CONFIG_ERROR ); # IPP camera configuration + # Grab the configuration data. my $conf_cmd = "$ppConfigDump -camera $camera -dump-recipe NIGHTLY_SCIENCE -"; @@ -155,20 +158,20 @@ foreach my $entry (@{ $metadata }) { if (@{ $entry }{name} eq 'CLEAN_MODES') { - my @mode_data = @{ ${ $entry }{value} }; - my $this_mode = ''; - foreach my $mentry (@mode_data) { - if (${ $mentry }{name} eq 'MODE') { - $this_mode = ${ $mentry }{value}; - } - elsif (${ $mentry }{name} eq 'COMMAND') { - $clean_config{$this_mode}{COMMAND} = ${ $mentry }{value}; - } - elsif (${ $mentry }{name} eq 'RETENTION_TIME') { - $clean_config{$this_mode}{RETENTION_TIME} = ${ $mentry }{value}; - } - } + my @mode_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@mode_data) { + if (${ $mentry }{name} eq 'MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'COMMAND') { + $clean_config{$this_mode}{COMMAND} = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'RETENTION_TIME') { + $clean_config{$this_mode}{RETENTION_TIME} = ${ $mentry }{value}; + } + } } elsif (${ $entry }{name} eq 'FILTERS') { - $unique_filter_hash{ ${ $entry }{value} } = 1; + $unique_filter_hash{ ${ $entry }{value} } = 1; # push @filter_list, ${ $entry }{value}; } @@ -177,14 +180,14 @@ } elsif (${ $entry }{name} eq 'MACRO_DEFINITIONS') { - my @macro_data = @{ ${ $entry }{value} }; - my $this_mode = ''; - foreach my $mentry (@macro_data) { - if (${ $mentry }{name} eq 'PROC_MODE') { - $this_mode = ${ $mentry }{value}; - } - elsif (${ $mentry }{name} eq 'MACRO') { - $macro_config{$this_mode} = ${ $mentry }{value}; - } - } + my @macro_data = @{ ${ $entry }{value} }; + my $this_mode = ''; + foreach my $mentry (@macro_data) { + if (${ $mentry }{name} eq 'PROC_MODE') { + $this_mode = ${ $mentry }{value}; + } + elsif (${ $mentry }{name} eq 'MACRO') { + $macro_config{$this_mode} = ${ $mentry }{value}; + } + } } elsif (${ $entry }{name} eq 'TARGETS') { @@ -194,34 +197,34 @@ if (${ $tentry }{name} eq 'NAME') { $this_target = ${ $tentry }{value}; - $science_config{$this_target}{DISTRIBUTION} = $this_target; # Set the default dist_group - } - else { - $science_config{$this_target}{ ${ $tentry }{name} } = ${ $tentry }{value}; - } + $science_config{$this_target}{DISTRIBUTION} = $this_target; # Set the default dist_group + } + else { + $science_config{$this_target}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } } } elsif (${ $entry }{name} eq 'DETRENDS') { - my @detrend_data = @{ ${ $entry }{value} }; - my $this_detrend = ''; - foreach my $dentry (@detrend_data) { - if (${ $dentry }{name} eq 'NAME') { - $this_detrend = ${ $dentry }{value}; - } - else { - $detrend_config{$this_detrend}{ ${ $dentry }{name} } = ${ $dentry }{value}; - } - } - } + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + } + else { + $detrend_config{$this_detrend}{ ${ $dentry }{name} } = ${ $dentry }{value}; + } + } + } elsif (${ $entry }{name} eq 'END_OF_NIGHT') { - my @eon_data = @{ ${ $entry }{value} }; - my $this_eon = ''; - foreach my $tentry (@eon_data) { - if (${ $tentry }{name} eq 'NAME') { - $this_eon = ${ $tentry }{value}; - } - else { - $eon_config{$this_eon}{ ${ $tentry }{name} } = ${ $tentry }{value}; - } - } + my @eon_data = @{ ${ $entry }{value} }; + my $this_eon = ''; + foreach my $tentry (@eon_data) { + if (${ $tentry }{name} eq 'NAME') { + $this_eon = ${ $tentry }{value}; + } + else { + $eon_config{$this_eon}{ ${ $tentry }{name} } = ${ $tentry }{value}; + } + } } } @@ -231,27 +234,27 @@ my $time; if ($date =~ / /) { - ($date,$time) = split / /, $date; + ($date,$time) = split / /, $date; } elsif ($date =~ /T/) { - ($date,$time) = split /T/, $date; + ($date,$time) = split /T/, $date; } my ($year,$month,$day) = split /-/, $date; if (defined($time)) { - my ($hour,$min,$sec) = split /\:/, $time; #/; - $datetime = DateTime->new( year => $year, - month => $month, - day => $day, - hour => $hour, - min => $min, - second => $sec, - time_zone => 'UTC'); + my ($hour,$min,$sec) = split /\:/, $time; #/; + $datetime = DateTime->new( year => $year, + month => $month, + day => $day, + hour => $hour, + min => $min, + second => $sec, + time_zone => 'UTC'); } else { - $datetime = DateTime->now(time_zone => 'UTC'); - $datetime->set_year($year); - $datetime->set_day($day); - $datetime->set_month($month); - } -} + $datetime = DateTime->now(time_zone => 'UTC'); + $datetime->set_year($year); + $datetime->set_day($day); + $datetime->set_month($month); + } +} else { $datetime = DateTime->now(time_zone => 'UTC'); # time_zone => 'Pacific/Honolulu'); @@ -263,5 +266,5 @@ foreach my $t (keys %science_config) { if ($t ne $this_target_only) { - undef($science_config{$t}); + undef($science_config{$t}); } } @@ -282,5 +285,5 @@ foreach my $t (keys %clean_config) { if ($t ne $this_mode_only) { - undef($clean_config{$t}); + undef($clean_config{$t}); } } @@ -295,10 +298,10 @@ if (defined($desdiffdt) == 0) { - $desdiffdt = 1.5; + $desdiffdt = 1.5; } #Build in a safety for desdiffdt if ((defined($desdiffdt) == 1) && ($desdiffdt <= 0.5)) { - $desdiffdt = 0.5; + $desdiffdt = 0.5; } @@ -312,23 +315,23 @@ my $v; foreach $z (sort (keys %science_config)) { - foreach $v (keys %{ $science_config{$z} }) { - print "SCIENCE: $z $v $science_config{$z}{$v}\n"; - } + foreach $v (keys %{ $science_config{$z} }) { + print "SCIENCE: $z $v $science_config{$z}{$v}\n"; + } } foreach $z (sort (keys %detrend_config)) { - foreach $v (keys %{ $detrend_config{$z} }) { - print "DETREND: $z $v $detrend_config{$z}{$v}\n"; - } + foreach $v (keys %{ $detrend_config{$z} }) { + print "DETREND: $z $v $detrend_config{$z}{$v}\n"; + } } foreach $z (keys (%clean_config)) { - foreach $v (keys %{ $clean_config{$z} }) { - print "CLEAN: $z $v $clean_config{$z}{$v}\n"; - } + foreach $v (keys %{ $clean_config{$z} }) { + print "CLEAN: $z $v $clean_config{$z}{$v}\n"; + } } foreach $z (@filter_list) { - print "FILTER: $z\n"; + print "FILTER: $z\n"; } foreach $z (keys (%macro_config)) { - print "MACROS: $z $macro_config{$z}\n"; + print "MACROS: $z $macro_config{$z}\n"; } } @@ -397,5 +400,5 @@ &execute_stacks($date,$metadata_out{nsObservingState}); if ($metadata_out{nsStackState} eq 'QUEUESTACKING') { - $metadata_out{nsStackState} = 'STACKING_POSSIBLE_ERROR'; + $metadata_out{nsStackState} = 'STACKING_POSSIBLE_ERROR'; } return_metadata($date); @@ -416,5 +419,5 @@ &execute_diffs($date,$metadata_out{nsObservingState}); if ($metadata_out{nsDiffState} eq 'QUEUESTACKING') { - $metadata_out{nsDiffState} = 'STACKING_POSSIBLE_ERROR'; + $metadata_out{nsDiffState} = 'STACKING_POSSIBLE_ERROR'; } return_metadata($date); @@ -481,9 +484,9 @@ my ($exp_name,$registered,$dateobs,$imfiles,$summit_fault, $download_state,$download_count,$new_state,$exp_id,$exp_type) = @{ $row_ref }; - $Nsummit_imfiles += $imfiles; - $Ndownload_imfiles += $download_count; + $Nsummit_imfiles += $imfiles; + $Ndownload_imfiles += $download_count; $Nsummit_exps++; if ($summit_fault) { - print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; + print STDERR "check_summit_copy: $date $exp_name has summit_fault $summit_fault"; if (($exp_type ne 'OBJECT')||($exp_name =~ /^c.*/)) { print STDERR " (but I don't care).\n"; @@ -526,6 +529,6 @@ } elsif ($Ndownload_imfiles != $Nsummit_imfiles) { - print STDERR "Not done downloading from the summit for $date (Summit: $Nsummit_imfiles, Downloaded: $Ndownload_imfiles)\n"; - $metadata_out{nsRegState} = 'NEW'; + print STDERR "Not done downloading from the summit for $date (Summit: $Nsummit_imfiles, Downloaded: $Ndownload_imfiles)\n"; + $metadata_out{nsRegState} = 'NEW'; } else { @@ -615,5 +618,5 @@ } if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; + $cmd .= " -set_reduction $reduction "; } $cmd .= " $select "; @@ -674,6 +677,6 @@ foreach my $target (sort (keys %science_config)) { - my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = check_chip_status($date,$target); - print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n"; + my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = check_chip_status($date,$target); + print "BTSTATS: $date $target $Nexposures $Nimfiles $Nburntooled $Nalready\n"; } } @@ -695,5 +698,5 @@ $cmd .= " -set_label dqstats.nightlyscience "; if ($debug == 1) { - $cmd .= ' -pretend '; + $cmd .= ' -pretend '; } print STDERR "$cmd\n"; @@ -703,5 +706,5 @@ sub pre_dqstats_queue { my $date = shift; - + my $command = construct_dqstats_cmd($date) . ' -pretend '; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -711,5 +714,5 @@ &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); } - + my @input_exposures = split /\n/, (join '', @$stdout_buf); @@ -735,19 +738,19 @@ my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); if ($Nexposures == 0) { - print STDERR "execute_dqstats: No exposures on date $date.\n"; + print STDERR "execute_dqstats: No exposures on date $date.\n"; } elsif ($Ncams != $Nchips) { - print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; } else { - unless(defined($pretend)) { - dqstats_queue($date); - } - if (defined($pretend)) { - add_to_macro_list('check_dqstats',1,$date); - } - else { - add_to_macro_list('queue_dqstats',1,$date); - } + unless(defined($pretend)) { + dqstats_queue($date); + } + if (defined($pretend)) { + add_to_macro_list('check_dqstats',1,$date); + } + else { + add_to_macro_list('queue_dqstats',1,$date); + } } } @@ -761,5 +764,5 @@ my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); - + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 "; @@ -778,12 +781,12 @@ $cmd .= " $use_limits "; if ($maxN > 0) { - $cmd .= " -random_subset -random_limit $maxN "; + $cmd .= " -random_subset -random_limit $maxN "; } if ($debug == 1) { - $cmd .= ' -pretend '; + $cmd .= ' -pretend '; } print STDERR "$cmd\n"; return($cmd); -} +} sub verify_uniqueness_detverify { @@ -792,5 +795,5 @@ my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); - + my $db = init_gpc_db(); my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; @@ -803,13 +806,13 @@ my $date = shift; my $target = shift; - + my $command = construct_dettool_cmd($date,$target) . ' -pretend '; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose); + run ( command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); - } - + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + my @input_exposures = split /\n/, (join '', @$stdout_buf); return($#input_exposures + 1); @@ -819,11 +822,11 @@ my $date = shift; my $target = shift; - + my $command = construct_dettool_cmd($date,$target); my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose); + run ( command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); } $metadata_out{nsDetState} = 'DETREND_QUEUED'; @@ -836,25 +839,25 @@ my $exposures = 0; foreach my $target (sort (keys %detrend_config)) { - my ($Nexposures) = pre_detrend_queue($date,$target); - if ($Nexposures == 0) { - print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; - next; - } - $exposures++; - unless(defined($pretend)) { - detrend_queue($date,$target); - } - if (defined($pretend)) { - add_to_macro_list('check_detrends',1,$date,$target,"dettool"); - } - else { - add_to_macro_list('queue_detrends',1,$date,$target,"dettool"); - } + my ($Nexposures) = pre_detrend_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; + next; + } + $exposures++; + unless(defined($pretend)) { + detrend_queue($date,$target); + } + if (defined($pretend)) { + add_to_macro_list('check_detrends',1,$date,$target,"dettool"); + } + else { + add_to_macro_list('queue_detrends',1,$date,$target,"dettool"); + } } if ($exposures == 0) { - $metadata_out{nsDetState} = 'DETREND_DROP'; + $metadata_out{nsDetState} = 'DETREND_DROP'; } if (($metadata_out{nsDetState} eq 'CHECKDETRENDS') && ($exposures > 0)) { - $metadata_out{nsDetState} eq 'QUEUEDETRENDS'; + $metadata_out{nsDetState} eq 'QUEUEDETRENDS'; } } @@ -882,5 +885,5 @@ # $cmd .= " $select "; # if ($debug == 1) { -# $cmd .= ' -pretend '; +# $cmd .= ' -pretend '; # } # print STDERR "$cmd\n"; @@ -895,46 +898,46 @@ my $dt = DateTime->new(year => $year, month => $month, day => $day, - hour => 0, minute => 0, second => 0, nanosecond => 0, - time_zone => 'Pacific/Honolulu'); + hour => 0, minute => 0, second => 0, nanosecond => 0, + time_zone => 'Pacific/Honolulu'); do { - $dt->add(days => 1); - my $ymd = $dt->ymd; - my $md_cmd = "moondata $ymd 0 0"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $md_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); - } - my @result = split /\s+/,(join "\n", @$stdout_buf); - if (abs($result[6]) <= 0.6) { - $dateobs_end = $ymd; - } + $dt->add(days => 1); + my $ymd = $dt->ymd; + my $md_cmd = "moondata $ymd 0 0"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $md_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + my @result = split /\s+/,(join "\n", @$stdout_buf); + if (abs($result[6]) <= 0.6) { + $dateobs_end = $ymd; + } } while (!defined($dateobs_end)); - + do { - $dt->subtract(days => 1); - my $ymd = $dt->ymd; - my $md_cmd = "moondata $ymd 0 0"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $md_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); - } - my @result = split /\s+/,(join "", @$stdout_buf); - if (abs($result[6]) <= 0.6) { - $dateobs_begin = $ymd; - } + $dt->subtract(days => 1); + my $ymd = $dt->ymd; + my $md_cmd = "moondata $ymd 0 0"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $md_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + my @result = split /\s+/,(join "", @$stdout_buf); + if (abs($result[6]) <= 0.6) { + $dateobs_begin = $ymd; + } } while (!defined($dateobs_begin)); - + return($dateobs_begin,$dateobs_end); } -# sub pre_sweetspot_queue { +# sub pre_sweetspot_queue { # my $date = shift; # my $command = construct_sweetspot_cmd($date) . ' -pretend '; # my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run ( command => $command, verbose => $verbose ); +# run ( command => $command, verbose => $verbose ); # unless ($success) { # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -943,5 +946,5 @@ # my @stacks = split /\n/, (join '', @$stdout_buf); # my $Nstacks = $#stacks + 1; - + # return(1,$Nstacks); # } @@ -951,5 +954,5 @@ # my $command = construct_sweetspot_cmd($date); # my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run ( command => $command, verbose => $verbose ); +# run ( command => $command, verbose => $verbose ); # unless ($success) { # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -958,39 +961,39 @@ # my @stacks = split /\n/, (join '', @$stdout_buf); # my $Nstacks = $#stacks + 1; - + # return(1,$Nstacks); # } -# sub execute_sweetspot { +# sub execute_sweetspot { # my $date = shift; # my $pretend = shift; - + # my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date); # if ($Nstacks == 0) { -# print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; -# $metadata_out{nsSSState} = 'SS_EMPTY'; -# return(); +# print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; +# $metadata_out{nsSSState} = 'SS_EMPTY'; +# return(); # } # if ($Nstacks < 10) { -# print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; -# $metadata_out{nsSSState} = 'SS_FEW'; -# return(); +# print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; +# $metadata_out{nsSSState} = 'SS_FEW'; +# return(); # } # if ($is_lunation_date == 0) { -# print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; -# $metadata_out{nsSSState} = 'SS_ERROR'; -# return(); +# print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; +# $metadata_out{nsSSState} = 'SS_ERROR'; +# return(); # } # $metadata_out{nsSSState} = 'SS_QUEUE'; # $metadata_out{nsSweetSpot} = $Nstacks; # unless(defined($pretend)) { -# $metadata_out{nsSSState} = 'SS_DONE'; -# sweetspot_queue($date); +# $metadata_out{nsSSState} = 'SS_DONE'; +# sweetspot_queue($date); # } # if (defined($pretend)) { -# add_to_macro_list('check_sweetspot',1,$date); +# add_to_macro_list('check_sweetspot',1,$date); # } # else { -# add_to_macro_list('queue_sweetspot',1,$date); +# add_to_macro_list('queue_sweetspot',1,$date); # } # } @@ -1020,33 +1023,33 @@ $cmd .= " -select_good_frac_min 0.05"; if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; + $cmd .= " -set_reduction $reduction "; } else { - $cmd .= " -set_reduction NIGHTLY_STACK "; + $cmd .= " -set_reduction NIGHTLY_STACK "; } if (defined($science_config{$target}{ADDITIONAL_STACK_LABEL})) { - # Grab list of skycells - my $skycell_select = ''; - { - my %skycells = (); - my $warpcmd = "$warptool -warped -label $label -data_group $data_group -dbname $dbname"; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $warpcmd, verbose => $verbose); - unless($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unabel to perform warptool -warped to determine skycell_ids: $error_code",$target); - } - my $runs = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unabel to parse warptool -warped to determine skycell_ids: $error_code",$target); - if ($#{ $runs } != -1) { - for my $warp (@$runs) { - $skycells{$warp->{skycell_id}} = 1; - } - foreach my $skycell (sort (keys (%skycells))) { - $skycell_select .= " -select_skycell_id $skycell "; - } - $cmd .= " -select_label $science_config{$target}{ADDITIONAL_STACK_LABEL} $skycell_select "; - } - } + # Grab list of skycells + my $skycell_select = ''; + { + my %skycells = (); + my $warpcmd = "$warptool -warped -label $label -data_group $data_group -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $warpcmd, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unabel to perform warptool -warped to determine skycell_ids: $error_code",$target); + } + my $runs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unabel to parse warptool -warped to determine skycell_ids: $error_code",$target); + if ($#{ $runs } != -1) { + for my $warp (@$runs) { + $skycells{$warp->{skycell_id}} = 1; + } + foreach my $skycell (sort (keys (%skycells))) { + $skycell_select .= " -select_skycell_id $skycell "; + } + $cmd .= " -select_label $science_config{$target}{ADDITIONAL_STACK_LABEL} $skycell_select "; + } + } } $cmd .= " $select "; @@ -1090,5 +1093,5 @@ my $where_possibly_faulted = $where . " AND ( " ; foreach my $acceptable_quality (@unrecoverable_quality) { - $where_possibly_faulted .= " quality = $acceptable_quality OR "; + $where_possibly_faulted .= " quality = $acceptable_quality OR "; } $where_possibly_faulted .= " 0 )"; @@ -1119,11 +1122,11 @@ $Nalready += $trash[0]; } - + my $minimum_required_warp; if ($observing_state eq 'OBSERVING') { - $minimum_required_warp = $science_config{$target}{MIN_STACK}; + $minimum_required_warp = $science_config{$target}{MIN_STACK}; } unless (defined($minimum_required_warp)) { - $minimum_required_warp = 0; + $minimum_required_warp = 0; } @@ -1143,6 +1146,6 @@ &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); } - - + + return(0); } @@ -1158,43 +1161,43 @@ my $is_registering; if ($metadata_out{nsRegistrationState} eq 'REGISTERED') { - $is_registering = 0; + $is_registering = 0; } else { - $is_registering = 1; + $is_registering = 1; } foreach my $target (sort (keys %science_config)) { - if ($science_config{$target}{STACKABLE} == 1) { - foreach my $filter (@filter_list) { + if ($science_config{$target}{STACKABLE} == 1) { + foreach my $filter (@filter_list) { my ($Nexposures,$NprocChips,$NprocWarps,$NrequiredWarps,$Nalready) = pre_stack_queue($date,$observing_state,$target,$filter); if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( - if ($debug == 1) { - print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; - } - $is_processing = 1; + if ($debug == 1) { + print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } + $is_processing = 1; $metadata_out{nsStackState} = 'FORCETOWARP'; next; } if ($Nexposures == 0) { - if ($debug == 1) { - print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; - } + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; + } next; } - if ($NprocWarps < $NrequiredWarps) { - if ($debug == 1) { - print STDERR "execute_stacks: Target $target in filter $filter on $date has too few warps to begin stacking. ($NprocWarps $NrequiredWarps)\n"; - } - next; - } - $Npotential++; + if ($NprocWarps < $NrequiredWarps) { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has too few warps to begin stacking. ($NprocWarps $NrequiredWarps)\n"; + } + next; + } + $Npotential++; if ($Nalready != 0) { - $Nqueued++; - if ($debug == 1) { - print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; - } - unless ($metadata_out{nsStackState} eq 'FORCETOWARP') { - $metadata_out{nsStackState} = 'STACKING'; - } + $Nqueued++; + if ($debug == 1) { + print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; + } + unless ($metadata_out{nsStackState} eq 'FORCETOWARP') { + $metadata_out{nsStackState} = 'STACKING'; + } next; } @@ -1203,22 +1206,22 @@ } unless(defined($pretend)) { - if ($debug == 1) { - print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; - } + if ($debug == 1) { + print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } stack_queue($date,$target,$filter); - $Nqueued ++; - } - if (defined($pretend)) { - add_to_macro_list('check_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); - } - else { - add_to_macro_list('queue_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); - } - } - } - else { - if ($debug == 1) { - print STDERR "execute_stacks: Target $target is not auto-stackable.\n"; - } + $Nqueued ++; + } + if (defined($pretend)) { + add_to_macro_list('check_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); + } + else { + add_to_macro_list('queue_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter); + } + } + } + else { + if ($debug == 1) { + print STDERR "execute_stacks: Target $target is not auto-stackable.\n"; + } } } @@ -1226,6 +1229,6 @@ $metadata_out{nsStackQueued} = $Nqueued; if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)&&($is_registering == 0)) { - $metadata_out{nsStackState} = 'FINISHED_STACKS'; - } + $metadata_out{nsStackState} = 'FINISHED_STACKS'; + } } @@ -1246,7 +1249,7 @@ $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) "; if ($debug == 1) { - print STDERR "post_stack_queue: database queries:\n"; - print STDERR "$stack_full_sth\n"; - print STDERR "$stack_done_sth\n"; + print STDERR "post_stack_queue: database queries:\n"; + print STDERR "$stack_full_sth\n"; + print STDERR "$stack_done_sth\n"; } my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth ); @@ -1259,37 +1262,37 @@ my $date = shift; my $pretend = shift; - + foreach my $target (sort (keys %science_config)) { if ($science_config{$target}{STACKABLE} == 1) { - my ($Nstacks, $Nattempted) = post_stack_queue($date,$target); - if ($debug == 1) { - print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n"; - } - if ($Nstacks != $Nattempted) { - if ($debug == 1) { - print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"; - } - if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { - $metadata_out{nsStackState} = 'STACKING'; - } - next; - } - if ($Nstacks == 0) { - if ($debug == 1) { - print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n"; - } - next; - } - if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { - if (defined($pretend)) { - add_to_macro_list('check_confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); - } - else { - add_to_macro_list('confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); - } - } - } - } -} + my ($Nstacks, $Nattempted) = post_stack_queue($date,$target); + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n"; + } + if ($Nstacks != $Nattempted) { + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"; + } + if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { + $metadata_out{nsStackState} = 'STACKING'; + } + next; + } + if ($Nstacks == 0) { + if ($debug == 1) { + print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n"; + } + next; + } + if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') { + if (defined($pretend)) { + add_to_macro_list('check_confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); + } + else { + add_to_macro_list('confirm_stacks',$science_config{$target}{STACKABLE},$date,$target); + } + } + } + } +} # @@ -1308,5 +1311,5 @@ my $count = 0; - + my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; my $data_ref = $db->selectall_arrayref( $sth ); @@ -1331,5 +1334,5 @@ my $count = 0; - + my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; my $data_ref = $db->selectall_arrayref( $sth ); @@ -1349,5 +1352,5 @@ my $filter = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); - + my $db = init_gpc_db(); @@ -1358,5 +1361,5 @@ my $where_possibly_faulted = $where . " AND ( " ; foreach my $acceptable_quality (@unrecoverable_quality) { - $where_possibly_faulted .= " quality = $acceptable_quality OR "; + $where_possibly_faulted .= " quality = $acceptable_quality OR "; } $where_possibly_faulted .= " 0 )"; @@ -1383,26 +1386,51 @@ my $is_registering; if ($metadata_out{nsRegistrationState} eq 'REGISTERED') { - $is_registering = 0; + $is_registering = 0; } else { - $is_registering = 1; + $is_registering = 1; } foreach my $target (sort (keys %science_config)) { - if ($science_config{$target}{DIFFABLE} == 1) { - foreach my $filter (@filter_list) { - $Npotential ++; - my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter); - if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) { - if ($debug == 1) { - print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n"; - } - $is_processing = 1; - } + if ($science_config{$target}{DIFFABLE} == 1) { + + my $config_reduction = "DEFAULT"; + if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} + print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; + + my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use + &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + + my $recipeData; + { + # Get the PSASTRO recipe + my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); + } + $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); + } + + my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); + print STDERR "execute_diffs: Max FWHM= $maxFWHM for $target\n"; + + foreach my $filter (@filter_list) { + $Npotential ++; + my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter); + if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) { + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n"; + } + $is_processing = 1; + } if ($NprocChips == 0) { - $Nnoexp ++; - if ($debug == 1) { - print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n"; - } + $Nnoexp ++; + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n"; + } next; } @@ -1410,75 +1438,99 @@ $metadata_out{nsDiffState} = 'QUEUEDIFFS'; } - if ($debug == 1) { - print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; - } - diff_queue($date,$target,$filter,$pretend); - - #Queue up desperate diffs if the conditions are right for them - if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { - desperate_diff_singles($date,$target,$filter,$pretend); - } - - if (defined($pretend)) { - add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); - } - else { - add_to_macro_list('queue_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); - } - } - } - else { - if ($debug == 1) { - print STDERR "execute_diffs: Target $target is not auto-diffable.\n"; - } + if ($debug == 1) { + print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n"; + } + diff_queue($date,$target,$filter,$maxFWHM,$pretend); + + #Queue up desperate diffs if the conditions are right for them + if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { + desperate_diff_singles($date,$target,$filter,$maxFWHM,$pretend); + } + + if (defined($pretend)) { + add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); + } + else { + add_to_macro_list('queue_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter); + } + } + } + else { + if ($debug == 1) { + print STDERR "execute_diffs: Target $target is not auto-diffable.\n"; + } } } if ($debug == 1) { - print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n"; + print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n"; } if ($metadata_out{nsObservingState} eq 'END_OF_NIGHT') { - if ($is_processing == 1) { - $metadata_out{nsDiffState} = 'DIFFING'; - } - elsif ($is_registering == 0) { - if ($Npotential == $Nnoexp) { - $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; - } - elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) { - $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; - } - } - else { - $metadata_out{nsDiffState} = 'DIFFING'; - } + if ($is_processing == 1) { + $metadata_out{nsDiffState} = 'DIFFING'; + } + elsif ($is_registering == 0) { + if ($Npotential == $Nnoexp) { + $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; + } + elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) { + $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; + } + } + else { + $metadata_out{nsDiffState} = 'DIFFING'; + } } else { - $metadata_out{nsDiffState} = 'DIFFING'; + $metadata_out{nsDiffState} = 'DIFFING'; } if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { - foreach my $target (sort (keys %science_config)) { - # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. - if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { - foreach my $filter (@filter_list) { - desperate_diff_queue($date,$target,$filter,$pretend); - } - } - - if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) { - foreach my $filter (@filter_list) { - # This one needs to return a state to see if we need to wait on stacking before checking again. - $metadata_out{nsDiffState} = self_WS_diff_queue($date,$target,$filter,$pretend); - } - } - } + foreach my $target (sort (keys %science_config)) { + + my $config_reduction = "DEFAULT"; + if($science_config{$target}{REDUCTION}) {$config_reduction=$science_config{$target}{REDUCTION};} + print STDERR "execute_diffs: Trying target $target on $date $camera $science_config{$target}{REDUCTION} $config_reduction\n"; + + my $recipe_psastro = $ipprcam->reduction($config_reduction, 'PSASTRO'); # Recipe to use + &my_die("Unrecognised PSASTRO recipe", $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + + my $recipeData; + { + # Get the PSASTRO recipe + my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $PS_EXIT_CONFIG_ERROR); + } + $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_CONFIG_ERROR); + } + + my $maxFWHM = metadataLookupF32($recipeData, 'PSASTRO.MAX.ALLOWED.FWHM'); + + # CZW: We should only arrive at this point once, so we can construct a diff report and mail it now. + if ((defined($science_config{$target}{DESPERATE_DIFFS})) && ($science_config{$target}{DESPERATE_DIFFS} == 1)) { + foreach my $filter (@filter_list) { + desperate_diff_queue($date,$target,$filter,$maxFWHM,$pretend); + } + } + + if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) { + foreach my $filter (@filter_list) { + # This one needs to return a state to see if we need to wait on stacking before checking again. + $metadata_out{nsDiffState} = self_WS_diff_queue($date,$target,$filter,$pretend); + } + } + } } # if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) { -# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; -# } +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } # if ($is_processing == 1) { -# $metadata_out{nsDiffState} = 'DIFFING'; +# $metadata_out{nsDiffState} = 'DIFFING'; # } } @@ -1488,4 +1540,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1500,135 +1553,152 @@ my $Npotential = 0; my $Nqueued = 0; - + foreach my $object_row (@{ $object_ref }) { - my $this_object = shift @{ $object_row }; - my $this_chunk = shift @{ $object_row }; - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; - $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; - $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; - $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; - - my $warps = $db->selectall_arrayref( $input_sth ); - - # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. - my %comment_hash = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - my $this_quality = ${ $this_warp }[5]; - if ($this_quality != 0) { - print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; - } - else { - $comment_hash{$this_comment} = $this_exp_id; - } - } - # Exclude any warps that are not stored in the comment_hash. - my @keep_warps = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { - push @keep_warps, $this_warp; - } - else { - print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; - } - } - @{ $warps } = @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; - if ($#{ $warps} + 1 == 1) { - print STDERR ": I can do no diffs with only one exposure.\n"; - next; - } - else { - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } - } - - while ($#{ $warps } > -1) { - # The array is sorted in pairs of input/template. - my $input_warp = shift @{ $warps }; - my $template_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } + my $this_object = shift @{ $object_row }; + my $this_chunk = shift @{ $object_row }; + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " AND camProcessedExp.fwhm_major <= $maxFWHM "; + $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY rawExp.comment,dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + my $Nbad = 0; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_quality = ${ $this_warp }[5]; + my $this_state = ${ $this_warp }[4]; + my $this_warp_id = ${ $this_warp }[1]; - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; -# $cmd .= " -pretend "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } + #find exposures for this object that have not been fully processed + if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) { + $Nbad += 1; + } + + if (($this_quality != 0) || ($this_state eq 'drop') ) { + print STDERR "diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state\n"; + } + else { + $comment_hash{$this_comment} = $this_exp_id; + } + } + + #kick object out of diff consideration if it has exposures not fully processed + if ($Nbad > 0) { + print STDERR "diff_queue: excluding $this_object from making diffs due to not being fully processed to warp stage\n"; + next; + } + + # Exclude any warps that are not stored in the comment_hash. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash{$this_comment}))&& + ($comment_hash{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + } + else { + print STDERR "diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; + } + } + @{ $warps } = @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + } + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $input_warp = shift @{ $warps }; + my $template_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } } $metadata_out{nsDiffPotential} += $Npotential; $metadata_out{nsDiffQueued} += $Nqueued; # if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { -# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; -# } +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } } @@ -1638,4 +1708,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1654,5 +1725,5 @@ my $Nqueued = 0; - #Before considering making diffs, we should check if there are no issues with summit download and/or registration. + #Before considering making diffs, we should check if there are no issues with summit download and/or registration. #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1 @@ -1672,5 +1743,5 @@ $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); - + $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); @@ -1685,58 +1756,82 @@ if (($summittimediff > 10.)||($noff >10)) { if ($debug == 1) { - print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; - } - } - else { + print STDERR "desp_diff_singles: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; + } + } + else { foreach my $object_row (@{ $object_ref }) { - my $this_object = shift @{ $object_row }; - my $this_chunk = shift @{ $object_row }; - - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)) FROM "; - $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; - $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; - $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; - - my $warps = $db->selectall_arrayref( $input_sth ); - - # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. - my %comment_hash = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - my $this_quality = ${ $this_warp }[5]; - if ($this_quality != 0) { - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; - } - else { - $comment_hash{$this_comment} = $this_exp_id; - } - } - # Exclude any warps that are not stored in the comment_hash. - my @keep_warps = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { - push @keep_warps, $this_warp; + my $this_object = shift @{ $object_row }; + my $this_chunk = shift @{ $object_row }; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,substr(comment, 1, position(' ' in comment)),camProcessedExp.fwhm_major FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY rawExp.comment,dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + my $warpsBQ = $db->selectall_arrayref( $input_sth );; + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + my %comment_hash_good = (); + my $Nbad = 0; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_quality = ${ $this_warp }[5]; + my $this_state = ${ $this_warp }[4]; + my $this_fwhm = ${ $this_warp }[7]; + my $this_warp_id = ${ $this_warp }[1]; + + #find exposures for this object that have not been fully processed + if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) { + $Nbad += 1; + } + + if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; + } + else { + $comment_hash_good{$this_comment} = $this_exp_id; + } + + #also save the entries which are ok, but not best quality + if (($this_quality == 0) && ($this_state ne 'drop') ) { + $comment_hash{$this_comment} = $this_exp_id; + } + } + + #kick object out of diff consideration if it has exposures not fully processed + if ($Nbad > 0) { + print STDERR "desp_diff_singles: excluding $this_object from making diffs due to not being fully processed to warp stage\n"; + next; + } + + # Exclude any warps that are not stored in the comment_hash_good. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash_good{$this_comment}))&& + ($comment_hash_good{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; $this_date = ${ $this_warp }[2]; $chunk_name = ${ $this_warp }[6]; - } - else { - print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; - } - } - @{ $warps } = @keep_warps; - + } + else { + print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + } + } + @{ $warps } = @keep_warps; + my $nwarps = ($#{ $warps } + 1); + #find the time of the most recent exposure in that chunk - my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; - my $chunk = $db->selectall_arrayref( $input_chunk ); - foreach my $this_chunk (@{ $chunk }) { - $this_date = ${ $this_chunk }[0]; + my $input_chunk = "select max(dateobs) FROM rawExp WHERE substr(comment, 1, position(' ' in comment)) = '$chunk_name' AND filter = '$filter' "; + my $chunk = $db->selectall_arrayref( $input_chunk ); + foreach my $this_chunk (@{ $chunk }) { + $this_date = ${ $this_chunk }[0]; } #compute and store some stats for potential on-the-fly desperate diffs - my $nwarps = ($#{ $warps } + 1); my $timediff = 0; my $now=DateTime->now; @@ -1750,100 +1845,245 @@ } } - - #Consider the special case of on-the-fly desperate diffs - #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old - if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { - print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; - next; - } - if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { - next; - } - - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } - - while ($#{ $warps } > -1) { - # The array is sorted in pairs of input/template. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } - - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_singles: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } + + ################################ + #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs + #Conversely, less than 3 good quality warps might result in dirty diffs + ################################ + if($nwarps >= 3) { + #Consider the special case of on-the-fly desperate diffs + #The conditions are: uneven nr of warps greater than 3, and the most recent observations is more than $desdiffdt hours old + if (($nwarps % 2 != 0) && ($nwarps >= 3) && ($timediff < $desdiffdt)) { + print STDERR "desp_diff_singles: There are potential desperate diffs to be done, but the time criterium is not met.\n"; + next; + } + if (($nwarps % 2 == 0) || ($nwarps == 1) || ($timediff < $desdiffdt)) { + next; + } + + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_singles: Number of input warps to make diffs is not even for target $target and object $this_object!"; + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_singles: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + + } else { + print STDERR "desp_diff_singles: There is a possibility for dirty diffs to be made\n"; + #Consider the case of on-the-fly dirty desperate diffs + #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old + +# # Exclude any warps that are not stored in the more general comment_hash. +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# if ((exists($comment_hash{$this_comment}))&& +# ($comment_hash{$this_comment} == $this_exp_id)) { +# push @keep_warps, $this_warp; +# $this_date = ${ $this_warp }[2]; +# $chunk_name = ${ $this_warp }[6]; +# } +# else { +# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# +# if ($nwarps >= 2) { +# #one regular diff has already been made. So, need only one more. Strip a good entry from the array +# my $nrem = 0; +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# my $this_fwhm = ${ $this_warp }[7]; +# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { +# $nrem++; +# print STDERR "desp_diff_singles: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; +# } else { +# push @keep_warps, $this_warp; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# } +# +# # We are attempting to do the missing diffs, so reverse the list of retained warps. +# # Good objects with all visits will be skipped due to the duplicate check. +# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. +# @{ $warpsBQ } = reverse @{ $warpsBQ }; +# +# # Exclude the last entry if we do not have an even number of warps. +# if (($#{ $warpsBQ } + 1) % 2 != 0) { +# print STDERR "desp_diff_singles: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; +# my $rejected_warp = pop @{ $warpsBQ }; +# my $rejected_exp_id = ${ $rejected_warp }[0]; +# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; +# } +# my $nwarpsBQ = ($#{$warpsBQ} + 1); +# +# +# if (($nwarpsBQ >= 2) && ($timediff < $desdiffdt)) { +# print STDERR "desp_diff_singles: There are potential dirty desperate diffs to be done, but the time criterium is not met.\n"; +# next; +# } +# if (($nwarpsBQ <= 1) || ($timediff < $desdiffdt)) { +# next; +# } +# +# while ($#{ $warpsBQ } > -1) { +# # The array is sorted in pairs of input/template. +# my $template_warp = shift @{ $warpsBQ }; +# my $input_warp = shift @{ $warpsBQ }; +# +# my $input_exp_id = ${ $input_warp }[0]; +# my $input_comment = ${ $input_warp }[3]; +# +# my $template_exp_id = ${ $template_warp }[0]; +# my $template_comment = ${ $template_warp }[3]; +# +# my $input_warp_id = ${ $input_warp }[1]; +# my $template_warp_id = ${ $template_warp }[1]; +# +# my $input_warp_state = ${ $input_warp }[4]; +# my $template_warp_state = ${ $template_warp }[4]; +# +# my $input_warp_camQuality = ${ $input_warp }[5]; +# my $template_warp_camQuality = ${ $template_warp }[5]; +# +# $Npotential++; +# +# unless (defined($input_warp_id) && defined($template_warp_id) && +# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { +# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; +# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { +# # This should now never be reached. +# # CZW: Trigger backup plan here? Or simply set up framework? +# print STDERR " ...but this is due to a camera stage astrometry quality\n"; +# $Npotential--; +# } +# next; +# } +# +# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { +# $Nqueued++; +# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# next; +# } +# else { +# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# } +# +# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; +# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; +# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. +# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures +# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; +# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# if (defined($reduction)) { +# $cmd .= " -set_reduction $reduction "; +# } +# +# if (defined($pretend)) { +# $cmd .= ' -pretend '; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# print STDERR "desp_diff_singles: $cmd\n"; +# print STDERR " $input_warp_id $template_warp_id\n"; +# } +# +# if (($debug == 0)&&(!defined($pretend))) { +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $cmd, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# $Nqueued++; +# } +# } + + + + } } } @@ -1856,4 +2096,5 @@ my $target = shift; my $filter = shift; + my $maxFWHM = shift; my $pretend = shift; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); @@ -1870,5 +2111,5 @@ my $Nqueued = 0; - #Before considering making diffs, we should check if there are no issues with summit download and/or registration. + #Before considering making diffs, we should check if there are no issues with summit download and/or registration. #summitExp has limited information, so see if the number of exps in rawExp and summitExp match up before most recent dateobs of chunk #be careful here, since dateobs is not quite completely the same in both tables. Therefore, allow numbers to be off by 1 @@ -1888,5 +2129,5 @@ $rawdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; my $date1=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); - + $summitdate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/; my $date2=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"UTC"); @@ -1901,145 +2142,306 @@ if (($summittimediff > 10.)||($noff >10)) { if ($debug == 1) { - print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; + print STDERR "desp_diff_queue: No desperate diffs will be attempted, since the number of exps at the summit and in rawExp do not match ($nsummitexps $nrawexps), or the timestamps are off ($summittimediff)\n"; } } else { foreach my $object_row (@{ $object_ref }) { - my $this_object = shift @{ $object_row }; - my $this_chunk = shift @{ $object_row }; - - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality FROM "; - $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; - $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; - $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY dateobs "; - - my $warps = $db->selectall_arrayref( $input_sth ); - - # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. - my %comment_hash = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - my $this_quality = ${ $this_warp }[5]; - if ($this_quality != 0) { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality\n"; - } - else { - $comment_hash{$this_comment} = $this_exp_id; - } - } - # Exclude any warps that are not stored in the comment_hash. - my @keep_warps = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ((exists($comment_hash{$this_comment}))&& - ($comment_hash{$this_comment} == $this_exp_id)) { - push @keep_warps, $this_warp; - } - else { - print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to not being an accepted comment string $this_comment\n"; - } - } - # We are attempting to do the missing diffs, so reverse the list of retained warps. - # Good objects with all visits will be skipped due to the duplicate check. - # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. - @{ $warps } = reverse @keep_warps; - - # Exclude the last entry if we do not have an even number of warps. - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; - if ($#{ $warps} + 1 == 1) { - print STDERR ": I can do no diffs with only one exposure.\n"; - next; - } - else { - my $rejected_warp = pop @{ $warps }; - my $rejected_exp_id = ${ $rejected_warp }[0]; - print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; - } - } - - while ($#{ $warps } > -1) { - # In this mode, the array is sorted in pairs of template/input, counter to the standard queue. - my $template_warp = shift @{ $warps }; - my $input_warp = shift @{ $warps }; - - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - my $template_exp_id = ${ $template_warp }[0]; - my $template_comment = ${ $template_warp }[3]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - my $input_warp_camQuality = ${ $input_warp }[5]; - my $template_warp_camQuality = ${ $template_warp }[5]; - - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; - if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { - # This should now never be reached. - # CZW: Trigger backup plan here? Or simply set up framework? - print STDERR " ...but this is due to a camera stage astrometry quality\n"; - $Npotential--; - } - next; - } - - if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - next; - } - else { - print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; - } + my $this_object = shift @{ $object_row }; + my $this_chunk = shift @{ $object_row }; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state,camProcessedExp.quality,camProcessedExp.fwhm_major FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN camProcessedExp USING(cam_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " AND substr(rawExp.comment, 1, position(' ' in rawExp.comment)) = '$this_chunk' ORDER BY rawExp.comment,dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + my $warpsBQ = $db->selectall_arrayref( $input_sth );; + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + my %comment_hash_good = (); + my $Nbad = 0; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + my $this_quality = ${ $this_warp }[5]; + my $this_state = ${ $this_warp }[4]; + my $this_fwhm = ${ $this_warp }[7]; + my $this_warp_id = ${ $this_warp }[1]; + + #find exposures for this object that have not been fully processed + if (($this_warp_id eq 'NULL') && (($this_quality == 0) || ($this_quality eq 'NULL')) ) { + $Nbad += 1; + } + + if (($this_quality != 0) || ($this_state eq 'drop') || ($this_fwhm > $maxFWHM)) { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to non-zero cam.quality $this_quality or state $this_state or FWHM $this_fwhm\n"; + } + else { + $comment_hash_good{$this_comment} = $this_exp_id; + } + + #also save the entries which are ok, but not best quality + if (($this_quality == 0) && ($this_state ne 'drop') ) { + $comment_hash{$this_comment} = $this_exp_id; + } + } + + #kick object out of diff consideration if it has exposures not fully processed + if ($Nbad > 0) { + print STDERR "desp_diff_queue: excluding $this_object from making diffs due to not being fully processed to warp stage\n"; + next; + } + + # Exclude any warps that are not stored in the comment_hash_good. + my @keep_warps = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ((exists($comment_hash_good{$this_comment}))&& + ($comment_hash_good{$this_comment} == $this_exp_id)) { + push @keep_warps, $this_warp; + my $this_date = ${ $this_warp }[2]; + my $chunk_name = ${ $this_warp }[6]; + } + else { + print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; + } + } + @{ $warps } = @keep_warps; + my $nwarps = ($#{ $warps } + 1); + + ################################ + #We reach a junction here, where having more than 3 (but uneven) warps will result in desperate diffs + #Conversely, less than 3 warps might result in dirty diffs + ################################ + if($nwarps >= 3) { + # We are attempting to do the missing diffs, so reverse the list of retained warps. + # Good objects with all visits will be skipped due to the duplicate check. + # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. + @{ $warps } = reverse @keep_warps; + + # Exclude the last entry if we do not have an even number of warps. + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "desp_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + my $rejected_warp = pop @{ $warps }; + my $rejected_exp_id = ${ $rejected_warp }[0]; + print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; + } + } + + while ($#{ $warps } > -1) { + # The array is sorted in pairs of input/template. + my $template_warp = shift @{ $warps }; + my $input_warp = shift @{ $warps }; + + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + my $template_exp_id = ${ $template_warp }[0]; + my $template_comment = ${ $template_warp }[3]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + my $input_warp_camQuality = ${ $input_warp }[5]; + my $template_warp_camQuality = ${ $template_warp }[5]; + + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; + if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { + # This should now never be reached. + # CZW: Trigger backup plan here? Or simply set up framework? + print STDERR " ...but this is due to a camera stage astrometry quality\n"; + $Npotential--; + } + next; + } + + if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + next; + } + else { + print STDERR "Preparing to diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; + } + + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "desp_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + + } else { + print STDERR "desp_diff_queue: There is a possibility for dirty diffs to be made\n"; + #Consider the case of on-the-fly dirty desperate diffs + #The conditions are: uneven nr of bad quality warps greater than 3, and the most recent observations is more than $desdiffdt hours old + +# # Exclude any warps that are not stored in the more general comment_hash. +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# if ((exists($comment_hash{$this_comment}))&& +# ($comment_hash{$this_comment} == $this_exp_id)) { +# push @keep_warps, $this_warp; +# my $this_date = ${ $this_warp }[2]; +# my $chunk_name = ${ $this_warp }[6]; +# } +# else { +# print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object due to being rejected $this_comment\n"; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# +# if ($nwarps >= 2) { +# #one regular diff has already been made. So, need only one more. Strip a good entry from the array +# my $nrem = 0; +# my @keep_warps = (); +# foreach my $this_warp (@{ $warpsBQ }) { +# my $this_comment = ${ $this_warp }[3]; +# my $this_exp_id = ${ $this_warp }[0]; +# my $this_fwhm = ${ $this_warp }[7]; +# if (($nrem <1) && ($this_fwhm <= $maxFWHM)) { +# $nrem++; +# print STDERR "desp_diff_queue: excluding $this_exp_id for $this_object from dirty desp diffs $this_comment\n"; +# } else { +# push @keep_warps, $this_warp; +# } +# } +# @{ $warpsBQ } = @keep_warps; +# } +# +# # We are attempting to do the missing diffs, so reverse the list of retained warps. +# # Good objects with all visits will be skipped due to the duplicate check. +# # Bad objects will have the earliest visit rejected, and the visits repaired in a way that will produce all desired pairs. +# @{ $warpsBQ } = reverse @{ $warpsBQ }; +# +# # Exclude the last entry if we do not have an even number of warps. +# if (($#{ $warpsBQ } + 1) % 2 != 0) { +# print STDERR "desp_diff_queue: Number of input warps to make dirty diffs is not even for target $target and object $this_object!"; +# my $rejected_warp = pop @{ $warpsBQ }; +# my $rejected_exp_id = ${ $rejected_warp }[0]; +# print STDERR ": Rejecting ${rejected_exp_id} to force visit count.\n"; +# } +# my $nwarpsBQ = ($#{$warpsBQ} + 1); +# +# +# while ($#{ $warpsBQ } > -1) { +# # The array is sorted in pairs of input/template. +# my $template_warp = shift @{ $warpsBQ }; +# my $input_warp = shift @{ $warpsBQ }; +# +# my $input_exp_id = ${ $input_warp }[0]; +# my $input_comment = ${ $input_warp }[3]; +# +# my $template_exp_id = ${ $template_warp }[0]; +# my $template_comment = ${ $template_warp }[3]; +# +# my $input_warp_id = ${ $input_warp }[1]; +# my $template_warp_id = ${ $template_warp }[1]; +# +# my $input_warp_state = ${ $input_warp }[4]; +# my $template_warp_state = ${ $template_warp }[4]; +# +# my $input_warp_camQuality = ${ $input_warp }[5]; +# my $template_warp_camQuality = ${ $template_warp }[5]; +# +# $Npotential++; +# +# unless (defined($input_warp_id) && defined($template_warp_id) && +# ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { +# print STDERR "Desp diff for this $date $target $input_exp_id $template_exp_id not fully processed ($input_warp_state $template_warp_state) ($input_warp_camQuality $template_warp_camQuality)\n"; +# if (($input_warp_camQuality == 4007)||($template_warp_camQuality == 4007)) { +# # This should now never be reached. +# # CZW: Trigger backup plan here? Or simply set up framework? +# print STDERR " ...but this is due to a camera stage astrometry quality\n"; +# $Npotential--; +# } +# next; +# } +# +# if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { +# $Nqueued++; +# print STDERR "Desp diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# next; +# } +# else { +# print STDERR "Preparing to dirty diff $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment $template_comment\n"; +# } +# +# my $cmd = "$difftool -dbname $dbname -definewarpwarp "; +# $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; +# $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. +# $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures +# $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; +# $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# if (defined($reduction)) { +# $cmd .= " -set_reduction $reduction "; +# } +# +# if (defined($pretend)) { +# $cmd .= ' -pretend '; +# } +# if ($debug == 1) { +# $cmd .= ' -pretend '; +# print STDERR "desp_diff_queue: $cmd\n"; +# print STDERR " $input_warp_id $template_warp_id\n"; +# } +# +# if (($debug == 0)&&(!defined($pretend))) { +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $cmd, verbose => $verbose ); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); +# } +# $Nqueued++; +# } +# } - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-rerun "; # Needed because we may have some diffs that already use some of the exposures - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; -# $cmd .= " -pretend "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "desp_diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } + + } } } $metadata_out{nsDiffPotential} += $Npotential; $metadata_out{nsDiffQueued} += $Nqueued; -# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { -# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; -# } - } @@ -2061,122 +2463,122 @@ my $Npotential = 0; my $Nqueued = 0; - + foreach my $object_row (@{ $object_ref }) { - my $this_object = shift @{ $object_row }; -# my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; -# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; -# $input_sth .= " ORDER BY dateobs "; - - my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM "; - $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; - $input_sth .= " WHERE warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; - $input_sth .= " ORDER BY dateobs "; - - my $warps = $db->selectall_arrayref( $input_sth ); - - # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. - my %comment_hash = (); - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - $comment_hash{$this_comment} = $this_exp_id; - } - - if (($#{ $warps } + 1) % 2 != 0) { - print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; - if ($#{ $warps} + 1 == 1) { - print STDERR ": I can do no diffs with only one exposure.\n"; - next; - } - else { - print STDERR ": I should declare an exposure to be faulty.\n"; - my @keep_warps = (); -# print "@{ $warps }\n"; - foreach my $this_warp (@{ $warps }) { - my $this_comment = ${ $this_warp }[3]; - my $this_exp_id = ${ $this_warp }[0]; - if ($comment_hash{$this_comment} == $this_exp_id) { - push @keep_warps, $this_warp; - } - else { - print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n"; - } - } - @{ $warps } = @keep_warps; -# print "@{ $warps }\n"; - } - } - - while ($#{ $warps } > -1) { - my $input_warp = shift @{ $warps }; - my $input_exp_id = ${ $input_warp }[0]; - my $input_comment = ${ $input_warp }[3]; - - - my $template_warp = shift @{ $warps }; - - my $template_exp_id = ${ $template_warp }[0]; - - my $input_warp_id = ${ $input_warp }[1]; - my $template_warp_id = ${ $template_warp }[1]; - - my $input_warp_state = ${ $input_warp }[4]; - my $template_warp_state = ${ $template_warp }[4]; - - unless(defined($template_warp)&& defined($template_exp_id)) { - print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n"; - next; - } - $Npotential++; - - unless (defined($input_warp_id) && defined($template_warp_id) && - ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { - print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n"; - next; - } - - if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { - $Nqueued++; - print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; - next; - } - - my $new_data_group = "${data_group}.multi"; - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; - $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. - $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $new_data_group "; - $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - -# $cmd .= " -pretend "; - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if ($debug == 1) { - $cmd .= ' -pretend '; - print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; - print STDERR "md_diff_queue: $cmd\n"; - print STDERR " $input_warp_id $template_warp_id\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - $Nqueued++; - } - } + my $this_object = shift @{ $object_row }; +# my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; +# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; +# $input_sth .= " ORDER BY dateobs "; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + $comment_hash{$this_comment} = $this_exp_id; + } + + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + print STDERR ": I should declare an exposure to be faulty.\n"; + my @keep_warps = (); +# print "@{ $warps }\n"; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ($comment_hash{$this_comment} == $this_exp_id) { + push @keep_warps, $this_warp; + } + else { + print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n"; + } + } + @{ $warps } = @keep_warps; +# print "@{ $warps }\n"; + } + } + + while ($#{ $warps } > -1) { + my $input_warp = shift @{ $warps }; + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + + my $template_warp = shift @{ $warps }; + + my $template_exp_id = ${ $template_warp }[0]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + unless(defined($template_warp)&& defined($template_exp_id)) { + print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n"; + next; + } + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n"; + next; + } + + if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + next; + } + + my $new_data_group = "${data_group}.multi"; + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label -good_frac 0.1 "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $new_data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + +# $cmd .= " -pretend "; + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + print STDERR "md_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } } $metadata_out{nsDiffPotential} += $Npotential; $metadata_out{nsDiffQueued} += $Nqueued; # if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { -# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; -# } +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } } @@ -2197,5 +2599,5 @@ -# Get a list of exposures that could be diffed +# Get a list of exposures that could be diffed my $new_data_group = "${data_group}.offnight"; my $new_dist_group = "${dist_group}.offnight"; @@ -2207,45 +2609,45 @@ $check_cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; if (defined($reduction)) { - $check_cmd .= " -set_reduction $reduction "; + $check_cmd .= " -set_reduction $reduction "; } my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $check_cmd, verbose => $verbose ); + run ( command => $check_cmd, verbose => $verbose ); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + # Parse results my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unable to parse metadata from difftool -definewarpwarp", 0, 0, $date, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata from difftool -definewarpwarp", 0, 0, $date, $PS_EXIT_PROG_ERROR); foreach my $diff (@$diffs) { - unless (multi_date_verify_uniqueness_diff($diff->{input_warp_id},$diff->{template_warp_id},$date,$target)) { - # If we don't already have a diff with these inputs, make a diff with these inputs. - my $cmd = "$difftool -dbname $dbname -definewarpwarp -good_frac 0.1 "; - $cmd .= "-input_label $label -template_label $label "; - $cmd .= "-warp_id $diff->{input_warp_id} -template_warp_id $diff->{template_warp_id} "; - $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; - $cmd .= " -mintimediff 40000 "; - $cmd .= " -simple -rerun -set_label $label -filter $filter "; - $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; - if (defined($reduction)) { - $cmd .= " -set_reduction $reduction "; - } - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - print STDERR "ON_diffs wants to run this command: $cmd\n"; - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } - } + unless (multi_date_verify_uniqueness_diff($diff->{input_warp_id},$diff->{template_warp_id},$date,$target)) { + # If we don't already have a diff with these inputs, make a diff with these inputs. + my $cmd = "$difftool -dbname $dbname -definewarpwarp -good_frac 0.1 "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-warp_id $diff->{input_warp_id} -template_warp_id $diff->{template_warp_id} "; + $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $cmd .= " -mintimediff 40000 "; + $cmd .= " -simple -rerun -set_label $label -filter $filter "; + $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + print STDERR "ON_diffs wants to run this command: $cmd\n"; + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } } } @@ -2261,11 +2663,11 @@ my ($stacks_to_finish,$stacks_total) = check_stack_count($label); if (($stacks_to_finish == 0)&&($stacks_total != 0)) { - my $cmd = "$difftool -dbname $dbname -definewarpstack "; - $cmd .= " -good_frac 0.2 "; - $cmd .= " -warp_label $label -stack_label $label -set_label $label "; - $cmd .= " -set_workdir $workdir -available -set_reduction WARPSTACK -set_dist_group $dist_group "; - $cmd .= " -rerun "; - } -} + my $cmd = "$difftool -dbname $dbname -definewarpstack "; + $cmd .= " -good_frac 0.2 "; + $cmd .= " -warp_label $label -stack_label $label -set_label $label "; + $cmd .= " -set_workdir $workdir -available -set_reduction WARPSTACK -set_dist_group $dist_group "; + $cmd .= " -rerun "; + } +} sub check_stack_count { @@ -2298,23 +2700,23 @@ my $retention_time; if (exists($science_config{$target}{$mode})) { - $retention_time = $science_config{$target}{$mode}; + $retention_time = $science_config{$target}{$mode}; } else { - $retention_time = $clean_config{$mode}{RETENTION_TIME}; + $retention_time = $clean_config{$mode}{RETENTION_TIME}; } if ($retention_time <= 0) { - return("no clean","true"); + return("no clean","true"); } my ($year,$month,$day) = split /-/,$date; my $dt = DateTime->new(year => $year, month => $month, day => $day, - hour => 0, minute => 0, second => 0, nanosecond => 0, - time_zone => 'Pacific/Honolulu'); - + hour => 0, minute => 0, second => 0, nanosecond => 0, + time_zone => 'Pacific/Honolulu'); + $dt->subtract(days => $retention_time); my $cleaning_date = $dt->ymd; my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); - + $command =~ s/\@DBNAME\@/$dbname/; $command =~ s/\@LABEL\@/$label/; @@ -2327,5 +2729,5 @@ $command =~ s/\@REDUCTION\@/$reduction/; $command =~ s/\@CURRENT_DATE\@/$cleaning_date/; - + if ($debug == 1) { $command .= ' -pretend '; @@ -2339,40 +2741,40 @@ foreach my $mode (sort (keys %clean_config)) { - foreach my $target (sort (keys %science_config)) { - if (exists($science_config{$target}{NOCLEAN})) { - next; - } - my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); - - if ($cleaning_date eq 'no clean') { - next; - } - print STDERR "$command\n"; - if (!(defined($pretend) || $debug == 1)) { + foreach my $target (sort (keys %science_config)) { + if (exists($science_config{$target}{NOCLEAN})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + + if ($cleaning_date eq 'no clean') { + next; + } + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { # print STDERR "BEAR IS DRIVING!?\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - add_to_macro_list('clean_old',1,$date,$target,$mode); - } - if ($mode eq 'DIFF') { - my $WS_command = $command; - $WS_command =~ s/$target/${target}.WS/; - print STDERR "$WS_command\n"; - if (!(defined($pretend) || $debug == 1)) { - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $WS_command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform cleantool ($WS_command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } - } - - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + add_to_macro_list('clean_old',1,$date,$target,$mode); + } + if ($mode eq 'DIFF') { + my $WS_command = $command; + $WS_command =~ s/$target/${target}.WS/; + print STDERR "$WS_command\n"; + if (!(defined($pretend) || $debug == 1)) { + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $WS_command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform cleantool ($WS_command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + + } } return(0); @@ -2386,40 +2788,40 @@ my $date = shift; my $eon_dt = DateTime->new( year => $datetime->year, - month => $datetime->month, - day => $datetime->day, - hour => 17, # 7, - minute => 30, - second => 0, - time_zone => 'UTC'); - + month => $datetime->month, + day => $datetime->day, + hour => 17, # 7, + minute => 30, + second => 0, + time_zone => 'UTC'); + foreach my $eon (keys %eon_config) { - my $command = "$regtool -processedexp -simple "; - $command .= " -dbname $dbname "; - $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; - $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); - $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); - $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); - $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - my @eon_exposures = split /\n/, (join '', @$stdout_buf); - if ($#eon_exposures >= 0) { - return("END_OF_NIGHT"); - } - } + my $command = "$regtool -processedexp -simple "; + $command .= " -dbname $dbname "; + $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); + $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); + $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); + $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @eon_exposures = split /\n/, (join '', @$stdout_buf); + if ($#eon_exposures >= 0) { + return("END_OF_NIGHT"); + } + } if ($force_registration) { - return("END_OF_NIGHT"); + return("END_OF_NIGHT"); } # print "$now $eon_dt " . DateTime->compare($now,$eon_dt) . "\n"; if (DateTime->compare($now,$eon_dt) < 1) { - return("OBSERVING"); + return("OBSERVING"); } else { - return("END_OF_NIGHT"); + return("END_OF_NIGHT"); } } @@ -2430,27 +2832,27 @@ foreach my $eon (keys %eon_config) { - my $command = "$regtool -processedexp -simple "; - $command .= " -dbname $dbname "; - $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; - $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); - $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); - $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); - $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $command, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - my @eon_exposures = split /\n/, (join '', @$stdout_buf); - if ($#eon_exposures >= 0) { - return("REGISTERED"); - } - } + my $command = "$regtool -processedexp -simple "; + $command .= " -dbname $dbname "; + $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; + $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT}); + $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE}); + $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE}); + $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT}); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + my @eon_exposures = split /\n/, (join '', @$stdout_buf); + if ($#eon_exposures >= 0) { + return("REGISTERED"); + } + } return("NOT_REGISTERED"); } - + sub get_tool_parameters { @@ -2486,12 +2888,12 @@ my $internal_filter; if (defined($det_filter)) { - $internal_filter = $det_filter; $internal_filter =~ s/\..*//; - $internal_filter = '.' . $internal_filter; + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; } else { - $internal_filter = ''; + $internal_filter = ''; } unless (defined($maxN)) { - $maxN = 0; + $maxN = 0; } my $lc_type = lc($exp_type); @@ -2509,36 +2911,36 @@ if ((defined($macro_config{$proc_mode}))&&($do_or_do_not)) { - unless (defined($metadata_out{N_MACROS})) { - $metadata_out{N_MACROS} = 0; - } - my $N = $metadata_out{N_MACROS}; - $metadata_out{"ns${N}Macro"} = $macro_config{$proc_mode}; - if ($debug == 1) { - print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; - } - if (defined($date)&&(defined($target))) { - - my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) - = get_tool_parameters($date,$target); - $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/; - $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/; - $metadata_out{"ns${N}Macro"} =~ s/\@OBSMODE\@/$obs_mode/; - $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/; - $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/; - $metadata_out{"ns${N}Macro"} =~ s/\@TESS_ID\@/$tess_id/; - $metadata_out{"ns${N}Macro"} =~ s/\@DIST_GROUP\@/$dist_group/; - $metadata_out{"ns${N}Macro"} =~ s/\@DATA_GROUP\@/$data_group/; - $metadata_out{"ns${N}Macro"} =~ s/\@REDUCTION\@/$reduction/; - } - if (defined($mode)) { - $metadata_out{"ns${N}Macro"} =~ s/\@EXTRA\@/$mode/; - } - if (defined($date)) { - $metadata_out{"ns${N}Macro"} =~ s/\@DATE\@/$date/; - } - if ($debug == 1) { - print STDERR "DONE WITH A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; - } - $metadata_out{N_MACROS} ++; + unless (defined($metadata_out{N_MACROS})) { + $metadata_out{N_MACROS} = 0; + } + my $N = $metadata_out{N_MACROS}; + $metadata_out{"ns${N}Macro"} = $macro_config{$proc_mode}; + if ($debug == 1) { + print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + if (defined($date)&&(defined($target))) { + + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) + = get_tool_parameters($date,$target); + $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/; + $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBSMODE\@/$obs_mode/; + $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/; + $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/; + $metadata_out{"ns${N}Macro"} =~ s/\@TESS_ID\@/$tess_id/; + $metadata_out{"ns${N}Macro"} =~ s/\@DIST_GROUP\@/$dist_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@DATA_GROUP\@/$data_group/; + $metadata_out{"ns${N}Macro"} =~ s/\@REDUCTION\@/$reduction/; + } + if (defined($mode)) { + $metadata_out{"ns${N}Macro"} =~ s/\@EXTRA\@/$mode/; + } + if (defined($date)) { + $metadata_out{"ns${N}Macro"} =~ s/\@DATE\@/$date/; + } + if ($debug == 1) { + print STDERR "DONE WITH A MACRO: ns${N}Macro $proc_mode $macro_config{$proc_mode}\n"; + } + $metadata_out{N_MACROS} ++; } }