Index: /trunk/ippScripts/scripts/automate_stacks.pl =================================================================== --- /trunk/ippScripts/scripts/automate_stacks.pl (revision 28426) +++ /trunk/ippScripts/scripts/automate_stacks.pl (revision 28427) @@ -17,4 +17,5 @@ my $missing_tools = 0; my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1); my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); @@ -28,22 +29,4 @@ exit($PS_EXIT_CONFIG_ERROR); } - -# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); -# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10', -# 'STS','SVS','SweetSpot','ThreePi'); -# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02', -# 'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06', -# 'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10', -# 'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0'); -# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%', -# 'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%', -# 'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%', -# 'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%'); -# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1, -# 'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1, -# 'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1, -# 'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0); -# my $retention_time = 9000; # days. - my $db; @@ -60,5 +43,5 @@ my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); -my ( $check_detrends, $queue_detrends); +my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); GetOptions( @@ -83,6 +66,8 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, - 'check_detrends' => \$check_detrends, - 'queue_detrends' => \$queue_detrends, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, + 'check_dqstats' => \$check_dqstats, + 'queue_dqstats' => \$queue_dqstats, 'check_diffs' => \$check_diffs, 'queue_diffs' => \$queue_diffs, @@ -111,4 +96,6 @@ --check_detrends Confirm that detrend verify runs can be built. --queue_detrends Issue dettool commands to queue detrend verify runs. + --check_dqstats Confirm that dqstats tables can be built. + --queue_dqstats Issue dqstatstool commands to queue dqstat tables. --check_diffs Confirm that diffs can be done. --queue_diffs Issue difftool commands to queue diffs. @@ -121,6 +108,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or $queue_detrends or - defined $check_chips or defined $check_stacks or $check_detrends or + defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or + defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode; @@ -132,4 +119,5 @@ my %object_list = (); my %comment_list= (); +my %cleanmods_list = (); my %stackable_list = (); my %reduction_class = (); @@ -144,4 +132,5 @@ my %clean_retention = (); my %noclean_list = (); +my %clean_alternate = (); # Grab the configuration data. my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; @@ -150,5 +139,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR); } @@ -169,4 +158,7 @@ $clean_retention{$this_mode} = ${ $mentry }{value}; } + elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') { + $clean_alternate{$this_mode} = ${ $mentry }{value}; + } } } @@ -202,4 +194,9 @@ elsif (${ $tentry }{name} eq 'NOCLEAN') { $noclean_list{$this_target} = ${ $tentry }{value}; + } + else { + if (exists($clean_commands{ ${ $tentry }{name} })) { + $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value}; + } } } @@ -292,5 +289,17 @@ unless (defined($test_mode) || defined($check_mode)) { exit(0); } } -if (defined($check_detrends) || defined($test_mode)) { +if (defined($check_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDQSTATS'; + &execute_dqstats($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_dqstats) || defined($test_mode)) { + $metadata_out{nsState} = 'QUEUEDQSTATS'; + &execute_dqstats($date); + return_metadata($date); + unless (defined($test_mode)) { exit(0); } +} +if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) { $metadata_out{nsState} = 'CHECKDETRENDS'; &execute_detrends($date,"pretend"); @@ -300,5 +309,5 @@ if (defined($queue_detrends)) { $metadata_out{nsState} = 'QUEUEDETRENDS'; - &execute_detrends($date,"pretend"); + &execute_detrends($date); return_metadata($date); exit(0); @@ -613,6 +622,4 @@ } - - sub chip_queue { my $date = shift; @@ -660,4 +667,80 @@ } +# +# DQstats +################################################################################ + +sub construct_dqstats_cmd { + my $date = shift; + + my $select = "-dateobs_end ${date}T23:59:59 "; + + my $cmd = "$dqstatstool"; + $cmd .= " -simple -dbname $dbname -definebyquery "; + $cmd .= " $select "; + $cmd .= " -label %.nightlyscience "; + $cmd .= " -set_label dqstats.nightlyscience "; + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub pre_dqstats_queue { + my $date = shift; + +# my $db = init_gpc_db(); +# my $trunc_date = $date; $trunc_date =~ s/-//g; + +# my $where = " label LIKE '%.nightlyscience' AND data_group' "; +# my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where "; +# my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where "; + +# my $chip_ref = $db->selectall_arrayref( $chip_sth ); +# my $cam_ref = $db->selectall_arrayref( $cam_sth ); + + my $command = construct_dqstats_cmd($date) . ' -pretend '; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + + return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1); +} + +sub dqstats_queue { + my $date = shift; + + my $command = construct_dqstatstool_cmd($date); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + return(0); + +} +sub execute_dqstats { + my $date = shift; + my $pretend = shift; + my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date); + if ($Nexposures == 0) { + print STDERR "execute_dqstats: No exposures on date $date.\n"; + } + elsif ($Ncams != $Nchips) { + print STDERR "execute_dqstats: Not done processing data through camera stage.\n"; + } + else { + unless(defined($pretend)) { + dqstats_queue($date); + } + } +} # # Detrend verification @@ -735,6 +818,6 @@ $metadata_out{nsState} = 'DETREND_QUEUED'; return(0); -} - + +} sub execute_detrends { my $date = shift; @@ -754,4 +837,7 @@ if ($exposures == 0) { $metadata_out{nsState} = 'DETREND_DROP'; + } + if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) { + $metadata_out{nsState} eq 'QUEUE_DETRENDS'; } } @@ -914,5 +1000,14 @@ my $command = $clean_commands{$mode}; - my $retention_time = $clean_retention{$mode}; + my $retention_time; + if (exists($cleanmods_list{$target}{$mode})) { + $retention_time = $cleanmods_list{$target}{$mode}; + } + else { + $retention_time = $clean_retention{$mode}; + } + if ($retention_time <= 0) { + return("no clean","true"); + } my ($year,$month,$day) = split /-/,$date; @@ -920,4 +1015,5 @@ hour => 0, minute => 0, second => 0, nanosecond => 0, time_zone => 'Pacific/Honolulu'); + $dt->subtract(days => $retention_time); my $cleaning_date = $dt->ymd; @@ -925,5 +1021,10 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args = $command; - $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date "; + } + else { + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + } if ($debug == 1) { $args .= ' -pretend '; @@ -936,11 +1037,10 @@ my $pretend = shift; - foreach my $mode (keys (%clean_commands)) { - foreach my $target (@target_list) { - if (exists($noclean_list{$target})) { + foreach my $mode (sort (keys (%clean_commands))) { + if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) { + my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); + if ($cleaning_date eq 'no clean') { next; } - my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); - print STDERR "$command\n"; if (!(defined($pretend) || $debug == 1)) { @@ -951,4 +1051,25 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + else { + foreach my $target (@target_list) { + if (exists($noclean_list{$target})) { + next; + } + my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode); + if ($cleaning_date eq 'no clean') { + next; + } + print STDERR "$command\n"; + if (!(defined($pretend) || $debug == 1)) { +# print STDERR "BEAR IS DRIVING!?\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } } } Index: /trunk/ippTasks/nightly_stacks.pro =================================================================== --- /trunk/ippTasks/nightly_stacks.pro (revision 28426) +++ /trunk/ippTasks/nightly_stacks.pro (revision 28427) @@ -6,4 +6,5 @@ ns.initday.on ns.detrends.off + ns.dqstats.on ns.registration.on ns.burntool.on @@ -51,4 +52,10 @@ end +macro ns.dqstats.on + task ns.dqstats.load + active true + end +end + macro ns.registration.on task ns.registration.load @@ -92,4 +99,10 @@ macro ns.detrends.off task ns.detrends.load + active false + end +end + +macro ns.dqstats.off + task ns.dqstats.load active false end @@ -227,4 +240,40 @@ task.exit 0 # nothign to do here + end + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + # operation times out + task.exit timeout + showcommand timeout + end +end + +# +# Queue dqstats runs +# +task ns.dqstats.load + host local + periods -poll 3600 + periods -exec $LOADEXEC + periods -timeout 30 + trange 22:00:00 23:59:59 + trange 00:00:00 10:00:00 + npending 1 + + task.exec + stdout $LOGDIR/ns.dqstats.log + stderr $LOGDIR/ns.dqstats.log + $today = `date +%Y-%m-%d` + + command automate_stacks.pl --queue_dqstats --date $today + end + + task.exit 0 + # nothing to do here end # locked list Index: /trunk/ippTools/src/dqstatstool.c =================================================================== --- /trunk/ippTools/src/dqstatstool.c (revision 28426) +++ /trunk/ippTools/src/dqstatstool.c (revision 28427) @@ -85,5 +85,5 @@ if (label) { - PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "LIKE"); // define using cam label } // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE @@ -654,5 +654,15 @@ psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); return(false); - } + } + psString filter = psMetadataLookupStr(&status,rule,"FILTER"); + if (!status) { + status = true; + } + if (filter) { + psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); + if (strcmp(filter,imfilter) != 0) { + continue; + } + } // Not happy with this being set to a F32. Can this ever be something else? psF32 value = psMetadataLookupF32(&status,tableRow,colname); Index: /trunk/ippconfig/recipes/dqStatsTool.config =================================================================== --- /trunk/ippconfig/recipes/dqStatsTool.config (revision 28426) +++ /trunk/ippconfig/recipes/dqStatsTool.config (revision 28427) @@ -7,10 +7,46 @@ COLNAME STR FWHM_MAJOR MINIMUM F32 0.0 - MAXIMUM F32 5.0 + MAXIMUM F32 9.3 # 90-percentile 2010-02-14 through 2010-06-20 = 2.4" END RULE METADATA COLNAME STR FWHM_MINOR MINIMUM F32 0.0 - MAXIMUM F32 5.0 + MAXIMUM F32 8.44 # 90-percentile 2010-02-14 through 2010-06-20 = 2.18" + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR g.00000 + MINIMUM F32 24.3051 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR r.00000 + MINIMUM F32 24.5294 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR i.00000 + MINIMUM F32 24.4641 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR z.00000 + MINIMUM F32 24.1128 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR y.00000 + MINIMUM F32 23.1503 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. + END + RULE METADATA + COLNAME STR ZEROPOINT + FILTER STR w.00000 + MINIMUM F32 25.8944 # (median 2010-02-14 through 2010-06-20) - 0.25 magnitudes + MAXIMUM F32 99.9999 # We refuse to believe in negative clouds. END # TYPE VAL COLNAME MINIMUM MAXIMUM Index: /trunk/ippconfig/recipes/nightly_science.config =================================================================== --- /trunk/ippconfig/recipes/nightly_science.config (revision 28426) +++ /trunk/ippconfig/recipes/nightly_science.config (revision 28427) @@ -6,4 +6,5 @@ COMMAND STR chiptool RETENTION_TIME U16 30 + ALTERNATE_CMD BOOL F END CLEAN_MODES METADATA @@ -11,10 +12,19 @@ COMMAND STR warptool RETENTION_TIME U16 7 -END -# CLEAN_MODES METADATA -# MODE STR DIFF -# COMMAND STR difftool -# RETENTION_TIME U16 30 -# END + ALTERNATE_CMD BOOL F +END +CLEAN_MODES METADATA + MODE STR DIFF + COMMAND STR difftool +# RETENTION_TIME U16 30 + RETENTION_TIME S16 -1 + ALTERNATE_CMD BOOL F +END +CLEAN_MODES METADATA + MODE STR DIST + COMMAND STR disttool + RETENTION_TIME S16 7 + ALTERNATE_CMD BOOL T +END FILTERS MULTI @@ -128,5 +138,6 @@ # OBJECT STR SS STACKABLE BOOL FALSE - NOCLEAN BOOL TRUE +# NOCLEAN BOOL TRUE + WARP S16 60 END TARGETS METADATA @@ -293,5 +304,5 @@ DETRENDS METADATA NAME STR DARK - DETTYPE STR DARK + DETTYPE STR DARK EXPTYPE STR dark REF_ID S64 299 @@ -303,5 +314,5 @@ DETRENDS METADATA NAME STR FLAT_G - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 300 @@ -313,5 +324,5 @@ DETRENDS METADATA NAME STR FLAT_R - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 301 @@ -323,5 +334,5 @@ DETRENDS METADATA NAME STR FLAT_I - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 302 @@ -333,5 +344,5 @@ DETRENDS METADATA NAME STR FLAT_Z - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 303 @@ -343,5 +354,5 @@ DETRENDS METADATA NAME STR FLAT_Y - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 304 @@ -353,5 +364,5 @@ DETRENDS METADATA NAME STR FLAT_W - DETTYPE STR FLAT + DETTYPE STR FLAT EXPTYPE STR skyflat REF_ID S64 305