Index: trunk/ippScripts/scripts/automate_stacks.pl
===================================================================
--- trunk/ippScripts/scripts/automate_stacks.pl	(revision 26745)
+++ trunk/ippScripts/scripts/automate_stacks.pl	(revision 26831)
@@ -9,10 +9,15 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
-
-
+use DateTime;
+
+#
 # Set up
+################################################################################
+
 my $missing_tools = 0;
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
@@ -23,7 +28,22 @@
 }
 
-# global variables.
-my @target_list = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10');
-my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
+# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
+# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
+# 		   'STS','SVS','SweetSpot','ThreePi');
+# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
+# 		   'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
+# 		   'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
+# 		   'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
+# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
+# 		    'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
+# 		    'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
+# 		    'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
+# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
+# 		    'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
+# 		    'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
+# 		    'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
+# my $retention_time = 9000;  # days.
+
+
 my $db;
 my $debug = 0;
@@ -33,8 +53,11 @@
 chomp($loghead = `date`);
 print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n";
+
 # Grab options
 my ( $date, $camera, $dbname, $logfile, $verbose);
-my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips, $check_stacks, $queue_stacks);
-my ( $help, $this_target_only, $this_filter_only, $isburning, $force_stack_count);
+my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only);
+my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
+my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
+
 GetOptions(
     'help|h'               => \$help,
@@ -44,15 +67,19 @@
     'logfile=s'            => \$logfile, 
     'verbose'              => \$verbose,
+    'isburning'            => \$isburning,
+    'force_stack_count'    => \$force_stack_count,
+    'test_mode'            => \$test_mode,
+    'this_target_only=s'   => \$this_target_only,
+    'this_filter_only=s'   => \$this_filter_only,
+    'check_registration'   => \$check_registration,
+    'define_burntool'      => \$define_burntool,
+    'queue_burntool'       => \$queue_burntool,
     'check_chips'          => \$check_chips,
     'queue_chips'          => \$queue_chips,
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
-    'queue_burntool'       => \$queue_burntool,
-    'define_burntool'      => \$define_burntool,
-    'check_registration|R' => \$check_registration,
-    'this_target_only=s'   => \$this_target_only,
-    'this_filter_only=s'   => \$this_filter_only,
-    'isburning'            => \$isburning,
-    'force_stack_count'    => \$force_stack_count,
+    'check_diffs'          => \$check_diffs,
+    'queue_diffs'          => \$queue_diffs,
+    'clean_old'            => \$clean_old,
     ) or pod2usage ( 2 );
 pod2usage( -msg => 
@@ -64,8 +91,8 @@
            --dbname <db>          Default gpc1.
            --verbose              
+           --isburning            Signal that we are currently burntooling.
+           --force_stack_count    Force the chip/warp counts.
            --this_target_only     Process only a single target.
            --this_filter_only     Process only a single filter.
-           --isburning            Signal that we are currently burntooling.
-           --force_stack_count    Force the chip/warp counts.
         Modes:
            --check_registration   Confirm the data downloaded correctly.
@@ -75,5 +102,8 @@
            --queue_chips          Issue chiptool commands to queue chips.
            --check_stacks         Confirm that stacks can be built.
-           --queue_stacks         Issue stacktool commands to queue stacks.\n",
+           --queue_stacks         Issue stacktool commands to queue stacks.
+           --check_diffs          Confirm that diffs can be done.
+           --queue_diffs          Issue difftool commands to queue diffs.
+           --clean_old            Mark old data 'goto_cleanup'.\n",
 	   -exitval => 2, ) if (defined($help));
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -84,5 +114,55 @@
     defined $check_registration or defined $define_burntool or defined $queue_burntool or
     defined $queue_chips or defined $queue_stacks or
-    defined $check_chips or defined $check_stacks;
+    defined $check_chips or defined $check_stacks or 
+    defined $test_mode or defined $clean_old;
+
+# Configurable parameters from our config file.
+my @target_list = ();
+my @filter_list = ();
+my %tessID_list = ();
+my %comment_list= ();
+my %stackable_list = ();
+my $retention_time = 9000;
+
+# Grab the configuration data.
+my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+    run(command => $conf_cmd, verbose => $verbose);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+foreach my $entry (@{ $metadata }) {
+    if (${ $entry }{name} eq 'RETENTION_TIME') {
+	$retention_time = ${ $entry }{value};
+    }
+    elsif (${ $entry }{name} eq 'FILTERS') {
+	push @filter_list, ${ $entry }{value};
+    }
+    elsif (${ $entry }{name} eq 'TARGETS') {
+	my @target_data = @{ ${ $entry }{value} };
+	my $this_target = '';
+	foreach my $tentry (@target_data) {
+	    if (${ $tentry }{name} eq 'NAME') {
+		$this_target = ${ $tentry }{value};
+		push @target_list, $this_target;
+	    }
+	    elsif (${ $tentry }{name} eq 'TESS') {
+		$tessID_list{$this_target} = ${ $tentry }{value};
+	    }
+	    elsif (${ $tentry }{name} eq 'COMMENT') {
+		$comment_list{$this_target} = ${ $tentry }{value};
+	    }
+	    elsif (${ $tentry }{name} eq 'STACKABLE') {
+		$stackable_list{$this_target} = ${ $tentry }{value};
+	    }
+	}
+    }
+}
+
+
 
 
@@ -120,34 +200,29 @@
 }
 
-if (defined($check_registration)) {
+#
+# Mode selection 
+################################################################################
+if (defined($test_mode)) {
+    $debug = 1;
+}
+
+if (defined($check_registration) || defined($test_mode)) {
+    $metadata_out{nsState} = 'NEW';
     my ($Nsummit_exp,$Nfaults) = check_summit_copy($date);
-    
-    if ($Nsummit_exp == 0) {
-	print STDERR "No exposures were found on the summit for $date.\n";
-	$metadata_out{nsState} = 'DROP';
-    }
-    elsif ($Nfaults != 0) {
-	print STDERR "There were faults while downloading the exposures for $date.\n";
-	$metadata_out{nsState} = 'NEW';
-    }
-    else {
-	print STDERR "Summit copy and Registration have succeeded for $date.\n";
-	$metadata_out{nsState} = 'REGISTERED';
-    }
     return_metadata($date);
-    exit(0);
-}
-if (defined($define_burntool)) {
+    unless (defined($test_mode)) { exit(0); }
+}
+if (defined($define_burntool) || defined($test_mode)) {
     $metadata_out{nsState} = 'QUEUEBURNING';
     &find_burntool_ranges($date);
     return_metadata($date);
-    exit(0);
-}
-if (defined($queue_burntool)) {
+    unless (defined($test_mode)) { exit(0); }
+}
+if (defined($queue_burntool) || defined($test_mode)) {
     $metadata_out{nsState} = 'BURNING';
     return_metadata($date);
-    exit(0);
-}
-if (defined($check_chips)) {
+    unless (defined($test_mode)) { exit(0); }
+}
+if (defined($check_chips) || defined($test_mode)) {
     $metadata_out{nsState} = 'QUEUECHIPS';
     &execute_chips($date,"pretend");
@@ -156,14 +231,5 @@
     }
     return_metadata($date);
-    exit(0);
-}
-if (defined($check_stacks)) {
-    $metadata_out{nsState} = 'TOWARP';
-    &execute_stacks($date,"pretend");
-    if ($metadata_out{nsState} eq 'FORCETOWARP') {
-	$metadata_out{nsState} = 'TOWARP';
-    }
-    return_metadata($date);
-    exit(0);
+    unless (defined($test_mode)) { exit(0); }
 }
 if (defined($queue_chips)) {
@@ -173,4 +239,13 @@
     exit(0);
 }
+if (defined($check_stacks) || defined($test_mode)) {
+    $metadata_out{nsState} = 'TOWARP';
+    &execute_stacks($date,"pretend");
+    if ($metadata_out{nsState} eq 'FORCETOWARP') {
+	$metadata_out{nsState} = 'TOWARP';
+    }
+    return_metadata($date);
+    unless (defined($test_mode)) { exit(0); }
+}
 if (defined($queue_stacks)) {
     $metadata_out{nsState} = 'STACKING';
@@ -179,6 +254,20 @@
     exit(0);
 }
-
+if (defined($check_diffs) || defined($queue_diffs)) {
+    die("Diffs are currently not implemented.");
+}
+if (defined($clean_old) || defined($test_mode)) {
+    if (defined($test_mode)) {
+	&execute_cleans($date,"pretend");
+    }
+    else {
+	&execute_cleans($date);
+    }
+    unless (defined($test_mode)) { exit(0); }
+}
+exit(0);
+#
 # Registration 
+################################################################################
 
 sub check_summit_copy {
@@ -240,9 +329,25 @@
 	}
     }
-    
-    return($Nsummit_exps,$Nsummit_faults + $Ndownload_faults + $Nregister_faults);
+
+    my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults;
+    if ($Nsummit_exps == 0) {
+	print STDERR "No exposures were found on the summit for $date.\n";
+	$metadata_out{nsState} = 'DROP';
+    }
+    elsif ($Nfaults != 0) {
+	print STDERR "There were faults while downloading the exposures for $date.\n";
+	$metadata_out{nsState} = 'NEW';
+    }
+    else {
+	print STDERR "Summit copy and Registration have succeeded for $date.\n";
+	$metadata_out{nsState} = 'REGISTERED';
+    }
+    
+    return($Nsummit_exps,$Nfaults);
 }	 
 
+#
 # Burntool
+################################################################################
 
 sub get_goodBTvalue {
@@ -325,10 +430,13 @@
 }
     
-
+#
 # Chips
+################################################################################
 
 sub construct_chiptool_cmd {
     my $date = shift;
     my $target = shift;
+
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
     
     my $select =  "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
@@ -337,7 +445,7 @@
     my $cmd = "$chiptool";
     $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
-    $cmd .= " -set_label ${target}.nightlystack ";
-    $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
-    $cmd .= " -set_tess_id $target -comment ${target}% -set_data_group ${target}.${date} ";
+    $cmd .= " -set_label $label ";
+    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
+    $cmd .= " -set_tess_id $tess_id -comment '$comment' -set_data_group $data_group ";
     $cmd .= " $select ";
     if ($debug == 1) {
@@ -355,5 +463,5 @@
     my $db = init_gpc_db();
     $date =~ s/-//g;
-    my $data_group = "${target}.${date}";
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
 
     my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id";
@@ -431,5 +539,7 @@
 }
 
+#
 # Stacking
+################################################################################
 
 sub construct_stacktool_cmd {
@@ -437,4 +547,6 @@
     my $target = shift;
     my $filter = shift;
+
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
 
     my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
@@ -444,7 +556,7 @@
 #    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
     $cmd .= ' -simple -dbname gpc1 -definebyquery ';  
-    $cmd .= " -set_label ${target}.nightlystack -select_label ${target}.nightlystack ";
-    $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
-    $cmd .= " -select_filter $filter -set_data_group ${target}.${date} ";
+    $cmd .= " -set_label $label -select_label $label ";
+    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
+    $cmd .= " -select_filter $filter -set_data_group $data_group ";
     $cmd .= " $select ";
     if ($debug == 1) {
@@ -463,5 +575,5 @@
     my $db = init_gpc_db();
     $date =~ s/-//g;
-    my $data_group = "${target}.${date}";
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
 
     my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'";
@@ -476,4 +588,5 @@
     my $filter = shift;
 
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
     # check warp stage == chip stage
     my $db = init_gpc_db();
@@ -481,5 +594,5 @@
     my $trunc_date = $date; $trunc_date =~ s/-//g;
 
-    my $where = " label = '${target}.nightlystack' AND data_group = '${target}.${trunc_date}' ";
+    my $where = " label = '$label' AND data_group = '$data_group' ";
     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
     my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
@@ -528,32 +641,115 @@
     my $pretend = shift;
     foreach my $target (@target_list) {
-	foreach my $filter (@filter_list) {
-	    my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
-	    if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
-		print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
-		$metadata_out{nsState} = 'FORCETOWARP';
-		next;
-	    }
-	    if ($Nexposures == 0) {
-		print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
-		next;
-	    }
-	    if ($Nalready != 0) {
-		print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
-		next;
-	    }
-	    unless ($metadata_out{nsState} eq 'FORCETOWARP') {
-		$metadata_out{nsState} = 'QUEUESTACKS';
-	    }
-	    unless(defined($pretend)) {
-		print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
-		stack_queue($date,$target,$filter);
-	    }
-	}
-    }
-    
-}
-
+	if ($stackable_list{$target} == 1) {
+	    foreach my $filter (@filter_list) {
+		my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
+		if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
+		    print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
+		    $metadata_out{nsState} = 'FORCETOWARP';
+		    next;
+		}
+		if ($Nexposures == 0) {
+		    print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
+		    next;
+		}
+		if ($Nalready != 0) {
+		    print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
+		    next;
+		}
+		unless ($metadata_out{nsState} eq 'FORCETOWARP') {
+		    $metadata_out{nsState} = 'QUEUESTACKS';
+		}
+		unless(defined($pretend)) {
+		    print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
+		    stack_queue($date,$target,$filter);
+		}
+	    }
+	}
+	else {
+	    print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+	}
+    }
+    
+}
+
+#
+# Auto-Clean
+################################################################################
+
+sub construct_cleantool_args {
+    my $date = shift;
+    my $target = shift;
+    
+    my ($year,$month,$day) = split /-/,$date;
+    my $dt = DateTime->new(year => $year, month => $month, day => $day,
+			       hour => 0, minute => 0, second => 0, nanosecond => 0,
+			       time_zone => 'Pacific/Honolulu');
+    $dt->subtract(days => $retention_time);
+    my $cleaning_date = $dt->ymd;
+
+    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($cleaning_date,$target);
+
+    my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group ";
+    if ($debug == 1) {
+	$args .= ' -pretend ';
+    }
+    return($cleaning_date,$args);
+}
+
+sub execute_cleans {
+    my $date = shift;
+    my $pretend = shift;
+
+    foreach my $target (@target_list) {
+	my ($cleaning_date,$args) = construct_cleantool_args($date,$target);
+
+	my $command = $chiptool . $args;
+	print STDERR "$command\n";
+	if (!(defined($pretend) || $debug == 1)) {
+#	    print STDERR "BEAR IS DRIVING!?\n";
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+		run ( command => $command, verbose => $verbose );
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+	    }
+	}
+	$command = $warptool . $args;
+	print STDERR "$command\n";
+	if (!(defined($pretend) || $debug == 1)) {
+#	    print STDERR "BEAR IS DRIVING $pretend $debug!?\n";
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+		run ( command => $command, verbose => $verbose );
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+	    }
+	}
+    }
+    return(0);
+}
+	
+	
+	    
+
+#
 # Utilities
+################################################################################
+
+sub get_tool_parameters {
+    my $date = shift;
+    my $target = shift;
+    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $label = "${target}.nightlyscience";
+    my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
+    my $comment = $comment_list{$target};
+    my $dist_group = $target;
+    my $data_group = "${target}.${trunc_date}";
+    my $tess_id = $tessID_list{$target};
+    return($label,$workdir,$comment,$tess_id,$dist_group,$data_group);
+}
+
 
 sub init_gpc_db {