Index: trunk/ippScripts/scripts/automate_stacks.pl
===================================================================
--- trunk/ippScripts/scripts/automate_stacks.pl	(revision 28388)
+++ trunk/ippScripts/scripts/automate_stacks.pl	(revision 28427)
@@ -17,4 +17,5 @@
 my $missing_tools = 0;
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $dqstatstool = can_run('dqstatstool') or (warn "Can't find dqstatstool" and $missing_tools = 1);
 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
@@ -28,22 +29,4 @@
     exit($PS_EXIT_CONFIG_ERROR);
 }
-
-# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
-# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
-#                  'STS','SVS','SweetSpot','ThreePi');
-# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
-#                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
-#                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
-#                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
-# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
-#                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
-#                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
-#                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
-# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
-#                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
-#                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
-#                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
-# my $retention_time = 9000;  # days.
-
 
 my $db;
@@ -60,5 +43,5 @@
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
-my ( $check_detrends, $queue_detrends);
+my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
 
 GetOptions(
@@ -83,6 +66,8 @@
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
-    'check_detrends'          => \$check_detrends,
-    'queue_detrends'          => \$queue_detrends,
+    'check_detrends'       => \$check_detrends,
+    'queue_detrends'       => \$queue_detrends,
+    'check_dqstats'        => \$check_dqstats,
+    'queue_dqstats'        => \$queue_dqstats,
     'check_diffs'          => \$check_diffs,
     'queue_diffs'          => \$queue_diffs,
@@ -111,4 +96,6 @@
            --check_detrends       Confirm that detrend verify runs can be built.
            --queue_detrends       Issue dettool commands to queue detrend verify runs.
+           --check_dqstats        Confirm that dqstats tables can be built.
+           --queue_dqstats        Issue dqstatstool commands to queue dqstat tables.
            --check_diffs          Confirm that diffs can be done.
            --queue_diffs          Issue difftool commands to queue diffs.
@@ -121,6 +108,6 @@
           ) unless
     defined $check_registration or defined $define_burntool or defined $queue_burntool or
-    defined $queue_chips or defined $queue_stacks or $queue_detrends or
-    defined $check_chips or defined $check_stacks or $check_detrends or
+    defined $queue_chips or defined $queue_stacks or $queue_detrends or $queue_dqstats or
+    defined $check_chips or defined $check_stacks or $check_detrends or $check_dqstats or
     defined $test_mode or defined $clean_old or defined $check_mode;
 
@@ -132,4 +119,5 @@
 my %object_list = ();
 my %comment_list= ();
+my %cleanmods_list = ();
 my %stackable_list = ();
 my %reduction_class = ();
@@ -144,4 +132,5 @@
 my %clean_retention = ();
 my %noclean_list = ();
+my %clean_alternate = ();
 # Grab the configuration data.
 my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
@@ -150,5 +139,5 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
 }
 
@@ -169,4 +158,7 @@
 		$clean_retention{$this_mode} = ${ $mentry }{value};
 	    }
+	    elsif (${ $mentry }{name} eq 'ALTERNATE_CMD') {
+		$clean_alternate{$this_mode} = ${ $mentry }{value};
+	    }
 	}
     }
@@ -202,4 +194,9 @@
 	    elsif (${ $tentry }{name} eq 'NOCLEAN') {
 		$noclean_list{$this_target} = ${ $tentry }{value};
+	    }
+	    else {
+		if (exists($clean_commands{ ${ $tentry }{name} })) {
+		    $cleanmods_list{$this_target}{${ $tentry }{name} } = ${ $tentry }{value};
+		}
 	    }
         }
@@ -292,5 +289,17 @@
     unless (defined($test_mode) || defined($check_mode)) { exit(0); }
 }
-if (defined($check_detrends) || defined($test_mode)) {
+if (defined($check_dqstats) || defined($test_mode)) {
+    $metadata_out{nsState} = 'CHECKDQSTATS';
+    &execute_dqstats($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($queue_dqstats) || defined($test_mode)) {
+    $metadata_out{nsState} = 'QUEUEDQSTATS';
+    &execute_dqstats($date);
+    return_metadata($date);
+    unless (defined($test_mode)) { exit(0); }
+}
+if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsState} = 'CHECKDETRENDS';
     &execute_detrends($date,"pretend");
@@ -300,5 +309,5 @@
 if (defined($queue_detrends)) {
     $metadata_out{nsState} = 'QUEUEDETRENDS';
-    &execute_detrends($date,"pretend");
+    &execute_detrends($date);
     return_metadata($date);
     exit(0);
@@ -613,6 +622,4 @@
 }
 
-
-
 sub chip_queue {
     my $date = shift;
@@ -660,4 +667,80 @@
 }
 
+#
+# DQstats
+################################################################################
+
+sub construct_dqstats_cmd {
+    my $date = shift;
+
+    my $select = "-dateobs_end ${date}T23:59:59 ";
+
+    my $cmd = "$dqstatstool";
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
+    $cmd .= " $select ";
+    $cmd .= " -label %.nightlyscience ";
+    $cmd .= " -set_label dqstats.nightlyscience ";
+    if ($debug == 1) {
+	$cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}
+
+sub pre_dqstats_queue {
+    my $date = shift;
+    
+#     my $db = init_gpc_db();
+#     my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+#     my $where = " label LIKE '%.nightlyscience' AND data_group' ";
+#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where ";
+#     my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where ";
+
+#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
+#     my $cam_ref = $db->selectall_arrayref( $cam_sth );
+    
+    my $command = construct_dqstats_cmd($date) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dqstatstool: $error_code",1,1,,$date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+
+    return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1);
+}
+ 
+sub dqstats_queue {
+    my $date = shift;
+
+    my $command = construct_dqstatstool_cmd($date);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dqstatstool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    return(0);
+
+}
+sub execute_dqstats {
+    my $date = shift;
+    my $pretend = shift;
+    my ($Nexposures,$Nchips,$Ncams) = pre_dqstats_queue($date);
+    if ($Nexposures == 0) {
+	print STDERR "execute_dqstats: No exposures on date $date.\n";
+    }
+    elsif ($Ncams != $Nchips) {
+	print STDERR "execute_dqstats: Not done processing data through camera stage.\n";
+    }
+    else {
+	unless(defined($pretend)) {
+	    dqstats_queue($date);
+	}
+    }
+}   
 #
 # Detrend verification
@@ -735,6 +818,6 @@
     $metadata_out{nsState} = 'DETREND_QUEUED';
     return(0);
-}
-
+
+}
 sub execute_detrends {
     my $date = shift;
@@ -754,4 +837,7 @@
     if ($exposures == 0) {
 	$metadata_out{nsState} = 'DETREND_DROP';
+    }
+    if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
+	$metadata_out{nsState} eq 'QUEUE_DETRENDS';
     }
 }
@@ -914,5 +1000,14 @@
 
     my $command = $clean_commands{$mode};
-    my $retention_time = $clean_retention{$mode};
+    my $retention_time;
+    if (exists($cleanmods_list{$target}{$mode})) {
+	$retention_time = $cleanmods_list{$target}{$mode};
+    }
+    else {
+	$retention_time = $clean_retention{$mode};
+    }
+    if ($retention_time <= 0) {
+	return("no clean","true");
+    }
 
     my ($year,$month,$day) = split /-/,$date;
@@ -920,4 +1015,5 @@
                                hour => 0, minute => 0, second => 0, nanosecond => 0,
                                time_zone => 'Pacific/Honolulu');
+	
     $dt->subtract(days => $retention_time);
     my $cleaning_date = $dt->ymd;
@@ -925,5 +1021,10 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
     my $args = $command;
-    $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
+    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+	$args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
+    }
+    else {
+	$args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
+    }
     if ($debug == 1) {
         $args .= ' -pretend ';
@@ -936,11 +1037,10 @@
     my $pretend = shift;
 
-    foreach my $mode (keys (%clean_commands)) {
-	foreach my $target (@target_list) {
-	    if (exists($noclean_list{$target})) {
+    foreach my $mode (sort (keys (%clean_commands))) {
+	if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+	    my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
+	    if ($cleaning_date eq 'no clean') {
 		next;
 	    }
-	    my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
-
 	    print STDERR "$command\n";
 	    if (!(defined($pretend) || $debug == 1)) {
@@ -951,4 +1051,25 @@
 		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
 		    &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		}
+	    }
+	}
+	else {
+	    foreach my $target (@target_list) {
+		if (exists($noclean_list{$target})) {
+		    next;
+		}
+		my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
+		if ($cleaning_date eq 'no clean') {
+		    next;
+		}		
+		print STDERR "$command\n";
+		if (!(defined($pretend) || $debug == 1)) {
+#           print STDERR "BEAR IS DRIVING!?\n";
+		    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+			run ( command => $command, verbose => $verbose );
+		    unless ($success) {
+			$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+			&my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		    }
 		}
 	    }