Index: branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl	(revision 30014)
+++ branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl	(revision 30017)
@@ -43,5 +43,5 @@
 # Grab options
 my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
-my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
+my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
 my ( $registration_status, $burntool_status, $observing_status);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
@@ -60,4 +60,5 @@
     'test_mode'            => \$test_mode,
     'force_stack_count'    => \$force_stack_count,
+    'force_diff_count'    => \$force_diff_count,
     'this_target_only=s'   => \$this_target_only,
     'this_filter_only=s'   => \$this_filter_only,
@@ -88,4 +89,5 @@
            --verbose
            --force_stack_count    Force the chip/warp counts.
+           --force_diff_count    Force the chip/warp counts.
            --this_target_only     Process only a single target.
            --this_filter_only     Process only a single filter.
@@ -232,6 +234,6 @@
 	$datetime = DateTime->now(time_zone => 'UTC');
 	$datetime->set_year($year);
-	$datetime->set_month($year);
-	$datetime->set_day($year);
+	$datetime->set_month($month);
+	$datetime->set_day($day);
     }
 }    
@@ -275,7 +277,9 @@
 }
 
+$metadata_out{nsObservingState} = &get_observing_state($date);
 #
 # Mode selection
 ################################################################################
+
 if (defined($test_mode)) {
     $debug = 1;
@@ -304,4 +308,10 @@
     }
 }
+if (defined($observing_status) || defined($test_mode)) {
+    $metadata_out{nsState} = 'CHECK_OBSERVING_STATUS';
+    return_metadata($date);
+    unless (defined($test_mode)) { exit(0); }
+}
+
 if (defined($registration_status) || defined($test_mode)) {
     &check_summit_copy($date);
@@ -314,9 +324,4 @@
     unless (defined($test_mode)) { exit(0); }
 }
-if (defined($observing_status) || defined($test_mode)) {
-    $metadata_out{nsState} = &get_observing_state($date);
-    return_metadata($date);
-    unless (defined($test_mode)) { exit(0); }
-}
 
 if (defined($check_dqstats) || defined($test_mode)) {
@@ -330,7 +335,7 @@
     unless (defined($test_mode)) { exit(0); }
 }
-exit(10);
+
 if (defined($check_detrends) || defined($test_mode)) {
-    $metadata_out{nsState} = 'CHECKDETRENDS';
+    $metadata_out{nsDetState} = 'CHECKDETRENDS';
     &execute_detrends($date,"pretend");
     return_metadata($date);
@@ -338,5 +343,5 @@
 }
 if (defined($queue_detrends)) {
-    $metadata_out{nsState} = 'QUEUEDETRENDS';
+    $metadata_out{nsDetState} = 'QUEUEDETRENDS';
     &execute_detrends($date);
     return_metadata($date);
@@ -344,9 +349,19 @@
 }
 
+if (defined($clean_old) || defined($test_mode)) {
+    if (defined($test_mode)) {
+        &execute_cleans($date,"pretend");
+    }
+    else {
+        &execute_cleans($date);
+    }
+    unless (defined($test_mode)) { exit(0); }
+}
+
 if (defined($check_stacks) || defined($test_mode)) {
-    $metadata_out{nsState} = 'TOWARP';
-    &execute_stacks($date,"pretend");
-    if ($metadata_out{nsState} eq 'FORCETOWARP') {
-        $metadata_out{nsState} = 'TOWARP';
+    $metadata_out{nsStackState} = 'TOWARP';
+    &execute_stacks($date,$metadata_out{nsObservingState},"pretend");
+    if ($metadata_out{nsStackState} eq 'FORCETOWARP') {
+        $metadata_out{nsStackState} = 'TOWARP';
     }
     return_metadata($date);
@@ -354,8 +369,8 @@
 }
 if (defined($queue_stacks)) {
-    $metadata_out{nsState} = 'STACKING';
-    &execute_stacks($date);
-    if ($metadata_out{nsState} eq 'QUEUESTACKING') {
-	$metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR';
+    $metadata_out{nsStackState} = 'STACKING';
+    &execute_stacks($date,$metadata_out{nsObservingState});
+    if ($metadata_out{nsStackState} eq 'QUEUESTACKING') {
+	$metadata_out{nsStackState} = 'STACKING_POSSIBLE_ERROR';
     }
     return_metadata($date);
@@ -364,8 +379,8 @@
 
 if (defined($check_diffs) || defined($test_mode)) {
-    $metadata_out{nsState} = 'TOWARP';
-    &execute_diffs($date,"pretend");
-    if ($metadata_out{nsState} eq 'FORCETOWARP') {
-        $metadata_out{nsState} = 'TOWARP';
+    $metadata_out{nsDiffState} = 'TOWARP';
+    &execute_diffs($date,$metadata_out{nsObservingState},"pretend");
+    if ($metadata_out{nsDiffState} eq 'FORCETOWARP') {
+        $metadata_out{nsDiffState} = 'TOWARP';
     }
     return_metadata($date);
@@ -373,15 +388,15 @@
 }
 if (defined($queue_diffs)) {
-    $metadata_out{nsState} = 'STACKING';
-    &execute_diffs($date);
-    if ($metadata_out{nsState} eq 'QUEUESTACKING') {
-	$metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR';
+    $metadata_out{nsDiffState} = 'DIFFING';
+    &execute_diffs($date,$metadata_out{nsObservingState});
+    if ($metadata_out{nsDiffState} eq 'QUEUESTACKING') {
+	$metadata_out{nsDiffState} = 'STACKING_POSSIBLE_ERROR';
     }
     return_metadata($date);
     exit(0);
 }
-
+exit(10);
 if (defined($check_confirm_stacks) || defined($test_mode)) {
-    $metadata_out{nsState} = 'CONFIRM_STACKING';
+    $metadata_out{nsStackState} = 'CONFIRM_STACKING';
     &confirm_stacks($date,"pretend");
     return_metadata($date);
@@ -389,5 +404,5 @@
 }
 if (defined($confirm_stacks)) {
-    $metadata_out{nsState} = 'CONFIRM_STACKING';
+    $metadata_out{nsStackState} = 'CONFIRM_STACKING';
     &confirm_stacks($date);
     return_metadata($date);
@@ -396,5 +411,5 @@
 
 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
-    $metadata_out{nsState} = 'CHECKSWEETSPOT';
+    $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
     &execute_sweetspot($date,"pretend");
     return_metadata($date);
@@ -402,5 +417,5 @@
 }
 if (defined($queue_sweetspot)) {
-    $metadata_out{nsState} = 'CHECKSWEETSPOT';
+    $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
     &execute_sweetspot($date);
     return_metadata($date);
@@ -408,13 +423,4 @@
 }
 
-if (defined($clean_old) || defined($test_mode)) {
-    if (defined($test_mode)) {
-        &execute_cleans($date,"pretend");
-    }
-    else {
-        &execute_cleans($date);
-    }
-    unless (defined($test_mode)) { exit(0); }
-}
 exit(0);
 #
@@ -488,17 +494,17 @@
     if ($Nsummit_exps == 0) {
         print STDERR "No exposures were found on the summit for $date.\n";
-        $metadata_out{nsState} = 'DROP';
+        $metadata_out{nsRegState} = 'DROP';
     }
     elsif ($Nfaults != 0) {
         print STDERR "There were faults while downloading the exposures for $date.\n";
-        $metadata_out{nsState} = 'NEW';
+        $metadata_out{nsRegState} = 'NEW';
     }
     elsif ($Ndownload_imfiles != $Nsummit_imfiles) {
 	print STDERR "Not done downloading from the summit for $date (Summit: $Nsummit_imfiles, Downloaded: $Ndownload_imfiles)\n";
-	$metadata_out{nsState} = 'NEW';
+	$metadata_out{nsRegState} = 'NEW';
     }
     else {
         print STDERR "Summit copy and Registration have succeeded for $date.\n";
-        $metadata_out{nsState} = 'REGISTERED';
+        $metadata_out{nsRegState} = 'REGISTERED';
     }
 
@@ -721,111 +727,111 @@
     }
 }   
-# #
-# # Detrend verification
-# ################################################################################
-
-# sub construct_dettool_cmd {
-#     my $date = shift;
-#     my $target = shift;
-
-#     my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
-    
-#     my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
-#     my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 ";
-#     $date =~ s/-//g;
-
-#     my $cmd = "$dettool";
-# #    $cmd .= " -pretend ";
-#     $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
-#     $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";
-#     $cmd .= " $select ";
-#     $cmd .= " -inst $camera ";
-#     $cmd .= " -select_exp_type $exp_type ";
-#     $cmd .= " -select_filter $filter " if defined($filter);
-#     $cmd .= " -workdir $workdir ";
-#     $cmd .= " -label $label ";
-#     $cmd .= " $use_limits ";
-#     if ($maxN > 0) {
-# 	$cmd .= " -random_subset -random_limit $maxN ";
-#     }
-#     if ($debug == 1) {
-# 	$cmd .= ' -pretend ';
-#     }
-#     print STDERR "$cmd\n";
-#     return($cmd);
-# }    
-
-# sub verify_uniqueness_detverify {
-#     my $date = shift;
-#     my $target = shift;
-
-#     my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
-    
-#     my $db = init_gpc_db();
-#     my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter";
-#     my $data_ref = $db->selectall_arrayref( $sth );
-
-#     return($#{ $data_ref } + 1);
-# }
-
-# sub pre_detrend_queue {
-#     my $date = shift;
-#     my $target = shift;
-    
-#     my $command = construct_dettool_cmd($date,$target) . ' -pretend ';
-#     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-# 	run ( command => $command, verbose => $verbose);
-#     unless ($success) {
-# 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-# 	&my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
-#     }
-    
-#     my @input_exposures = split /\n/, (join '', @$stdout_buf);
-#     return($#input_exposures + 1);
-# }
-
-# sub detrend_queue {
-#     my $date = shift;
-#     my $target = shift;
-    
-#     my $command = construct_dettool_cmd($date,$target);
-#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-# 	run ( command => $command, verbose => $verbose);
-#     unless ($success) {
-# 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-# 	&my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-#     }
-#     $metadata_out{nsState} = 'DETREND_QUEUED';
-#     return(0);
-
-# }
-# sub execute_detrends {
-#     my $date = shift;
-#     my $pretend = shift;
-#     my $exposures = 0;
-#     foreach my $target (@detrend_list) {
-# 	my ($Nexposures) = pre_detrend_queue($date,$target);
-# 	if ($Nexposures == 0) {
-# 	    print STDERR "execute_detrends: Target $target on $date had no exposures.\n";
-# 	    next;
-# 	}
-# 	$exposures++;
-# 	unless(defined($pretend)) {
-# 	    detrend_queue($date,$target);
-# 	}
-# 	if (defined($pretend)) {
-# 	    add_to_macro_list('check_detrends',1,$date,$target,"dettool");
-# 	}
-# 	else {
-# 	    add_to_macro_list('queue_detrends',1,$date,$target,"dettool");
-# 	}
-#     }
-#     if ($exposures == 0) {
-# 	$metadata_out{nsState} = 'DETREND_DROP';
-#     }
-#     if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
-# 	$metadata_out{nsState} eq 'QUEUEDETRENDS';
-#     }
-# }
+#
+# Detrend verification
+################################################################################
+
+sub construct_dettool_cmd {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$dettool";
+#    $cmd .= " -pretend ";
+    $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
+    $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";
+    $cmd .= " $select ";
+    $cmd .= " -inst $camera ";
+    $cmd .= " -select_exp_type $exp_type ";
+    $cmd .= " -select_filter $filter " if defined($filter);
+    $cmd .= " -workdir $workdir ";
+    $cmd .= " -label $label ";
+    $cmd .= " $use_limits ";
+    if ($maxN > 0) {
+	$cmd .= " -random_subset -random_limit $maxN ";
+    }
+    if ($debug == 1) {
+	$cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}    
+
+sub verify_uniqueness_detverify {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $db = init_gpc_db();
+    my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target) . ' -pretend ';
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+    return($#input_exposures + 1);
+}
+
+sub detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    $metadata_out{nsDetState} = 'DETREND_QUEUED';
+    return(0);
+
+}
+sub execute_detrends {
+    my $date = shift;
+    my $pretend = shift;
+    my $exposures = 0;
+    foreach my $target (sort (keys %detrend_config)) {
+	my ($Nexposures) = pre_detrend_queue($date,$target);
+	if ($Nexposures == 0) {
+	    print STDERR "execute_detrends: Target $target on $date had no exposures.\n";
+	    next;
+	}
+	$exposures++;
+	unless(defined($pretend)) {
+	    detrend_queue($date,$target);
+	}
+	if (defined($pretend)) {
+	    add_to_macro_list('check_detrends',1,$date,$target,"dettool");
+	}
+	else {
+	    add_to_macro_list('queue_detrends',1,$date,$target,"dettool");
+	}
+    }
+    if ($exposures == 0) {
+	$metadata_out{nsDetState} = 'DETREND_DROP';
+    }
+    if (($metadata_out{nsDetState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
+	$metadata_out{nsDetState} eq 'QUEUEDETRENDS';
+    }
+}
 
 # #
@@ -938,21 +944,21 @@
 #     if ($Nstacks == 0) {
 # 	print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n";
-# 	$metadata_out{nsState} = 'SS_EMPTY';
+# 	$metadata_out{nsSSState} = 'SS_EMPTY';
 # 	return();
 #     }
 #     if ($Nstacks < 10) {
 # 	print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n";
-# 	$metadata_out{nsState} = 'SS_FEW';
+# 	$metadata_out{nsSSState} = 'SS_FEW';
 # 	return();
 #     }
 #     if ($is_lunation_date == 0) {
 # 	print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n";
-# 	$metadata_out{nsState} = 'SS_ERROR';
+# 	$metadata_out{nsSSState} = 'SS_ERROR';
 # 	return();
 #     }
-#     $metadata_out{nsState} = 'SS_QUEUE';
+#     $metadata_out{nsSSState} = 'SS_QUEUE';
 #     $metadata_out{nsSweetSpot} = $Nstacks;
 #     unless(defined($pretend)) {
-# 	$metadata_out{nsState} = 'SS_DONE';
+# 	$metadata_out{nsSSState} = 'SS_DONE';
 # 	sweetspot_queue($date);
 #     }
@@ -965,334 +971,412 @@
 # }
 
-# #
-# # Stacking
-# ################################################################################
-
-# sub construct_stacktool_cmd {
-#     my $date = shift;
-#     my $target = shift;
-#     my $filter = shift;
-
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-
-#     my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
-#     $date =~ s/-//g;
-
-#     my $cmd = "$stacktool";
-# #    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
-#     $cmd .= " -simple -dbname $dbname -definebyquery ";
-#     $cmd .= " -set_label $label -select_label $label ";
-#     $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
-#     $cmd .= " -select_filter $filter -set_data_group $data_group ";
-#     $cmd .= " -min_num 4";
-#     $cmd .= " -select_good_frac_min 0.05";
-#     $cmd .= " $select ";
-#     if ($debug == 1) {
-#         $cmd .= ' -pretend ';
-#     }
-#     print STDERR "$cmd\n";
-#     return($cmd);
-# }
-
-# sub verify_uniqueness_stack {
-#     my $skycell = shift;
-#     my $date = shift;
-#     my $target = shift;
-#     my $filter = shift;
-
-#     my $db = init_gpc_db();
-#     $date =~ s/-//g;
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-
-#     my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'";
-#     my $data_ref = $db->selectall_arrayref( $sth );
-
-#     return($#{ $data_ref } + 1);
-# }
-
-# sub pre_stack_queue {
-#     my $date = shift;
-#     my $target = shift;
-#     my $filter = shift;
-
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-#     # check warp stage == chip stage
-#     my $db = init_gpc_db();
-
-#     my $trunc_date = $date; $trunc_date =~ s/-//g;
-
-#     my $where = " label = '$label' AND data_group = '$data_group' ";
-
-#     my $where_possibly_faulted = $where . " AND ( " ;
-#     foreach my $acceptable_quality (@unrecoverable_quality) {
-# 	$where_possibly_faulted .= " quality = $acceptable_quality OR ";
-#     }
-#     $where_possibly_faulted .= " 0 )";
-
-#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
-#     my $cam_sth  = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted ";
-#     my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
-
-#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
-#     my $cam_ref  = $db->selectall_arrayref( $cam_sth );
-#     my $warp_ref = $db->selectall_arrayref( $warp_sth );
-
-#     # check that we will be able to queue them up
-#     my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend ';
-#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#         run ( command => $command, verbose => $verbose );
-#     unless ($success) {
-#         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-#         &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR);
-#     }
-
-#     my $Nalready = 0;
-
-#     my @input_exposures = split /\n/, (join '', @$stdout_buf);
-#     foreach my $entry (@input_exposures) {
-#         my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry;
-#         @trash = verify_uniqueness_stack($skycell,$date,$target,$filter);
-#         $Nalready += $trash[0];
-#     }
-
-#     return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $Nalready);
-# }
-
-# sub stack_queue {
-#     my $date = shift;
-#     my $target = shift;
-#     my $filter = shift;
-
-#     my $command = construct_stacktool_cmd($date,$target,$filter);
-#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#         run ( command => $command, verbose => $verbose );
-#     unless ($success) {
-#         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-#         &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-#     }
-    
-    
-#     return(0);
-# }
-
-# sub execute_stacks {
-#     my $date = shift;
-#     my $pretend = shift;
-#     foreach my $target (@target_list) {
-# 	foreach my $filter (@filter_list) {
-# 	    if (exists($extra_processing{$target})) {
-#                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
-#                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
-# 		    next;
-# 		}
-# 		if ($Nexposures == 0) {
-# 		    if ($debug == 1) {
-# 			print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
-# 		    }
-#                     next;
-# 		}		    
-# 		else {
-# 		    do_extra_processing($date,$target,$filter,$pretend);
-# 		}
-
-# 	    }
-# 	    if ($science_config{$target}{STACKABLE} == 1) {
-#                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
-#                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
-# 		    if ($debug == 1) {
-# 			print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
-# 		    }
-#                     $metadata_out{nsState} = 'FORCETOWARP';
-#                     next;
-#                 }
-#                 if ($Nexposures == 0) {
-# 		    if ($debug == 1) {
-# 			print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
-# 		    }
-#                     next;
-#                 }
-#                 if ($Nalready != 0) {
-# 		    if ($debug == 1) {
-# 			print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
-# 		    }
-# 		    unless ($metadata_out{nsState} eq 'FORCETOWARP') {
-# 			$metadata_out{nsState} = 'STACKING';
-# 		    }
-#                     next;
-#                 }
-#                 unless (($metadata_out{nsState} eq 'FORCETOWARP')||($metadata_out{nsState} eq 'STACKING')) {
-#                     $metadata_out{nsState} = 'QUEUESTACKS';
-#                 }
-#                 unless(defined($pretend)) {
-# 		    if ($debug == 1) {
-# 			print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
-# 		    }
-#                     stack_queue($date,$target,$filter);
-#                 }
-# 		if (defined($pretend)) {
-# 		    add_to_macro_list('check_stacks',$stackable_list{$target},$date,$target,$filter);
-# 		}
-# 		else {
-# 		    add_to_macro_list('queue_stacks',$stackable_list{$target},$date,$target,$filter);
-# 		}
-#             }
-# 	    else {
-# 		# print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
-# 	    }
-#         }
-#     }
-# }
-
-# sub post_stack_queue {
-#     my $date = shift;
-#     my $target = shift;
-
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-#     # check warp stage == chip stage
-#     my $db = init_gpc_db();
-
-#     my $trunc_date = $date; $trunc_date =~ s/-//g;
-
-#     my $where = " label = '$label' AND data_group = '$data_group' ";
-
-#     my $stack_full_sth = "SELECT * from stackRun where $where ";
-#     my $stack_done_sth = "SELECT * from stackRun LEFT JOIN stackSumSkyfile USING(stack_id) WHERE $where ";
-#     $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) ";
-#     if ($debug == 1) {
-# 	print STDERR "post_stack_queue: database queries:\n";
-# 	print STDERR "$stack_full_sth\n";
-# 	print STDERR "$stack_done_sth\n";
-#     }
-#     my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth );
-#     my $stack_done_ref = $db->selectall_arrayref( $stack_done_sth );
-
-#     return($#{ $stack_full_ref } + 1, $#{ $stack_done_ref } + 1);
-# }
-
-# sub confirm_stacks {
-#     my $date = shift;
-#     my $pretend = shift;
-    
-#     foreach my $target (@target_list) {
-#         if ($stackable_list{$target} == 1) {
-# 	    my ($Nstacks, $Nattempted) = post_stack_queue($date,$target);
-# 	    if ($debug == 1) {
-# 		print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n";
-# 	    }
-# 	    if ($Nstacks != $Nattempted) {
-# 		if ($debug == 1) {
-# 		    print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n";
-# 		}
-# 		if ($metadata_out{nsState} eq 'CONFIRM_STACKING') {
-# 		    $metadata_out{nsState} = 'STACKING';
-# 		}
-# 		next;
-# 	    }
-# 	    if ($Nstacks == 0) {
-# 		if ($debug == 1) {
-# 		    print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n";
-# 		}
-# 		next;
-# 	    }
-# 	    if ($metadata_out{nsState} eq 'CONFIRM_STACKING') {
-# 		if (defined($pretend)) {
-# 		    add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target);
-# 		}
-# 		else {
-# 		    add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target);
-# 		}
-# 	    }
-# 	}
-#     }
-# }	    
-
-# #
-# # Extra processing
-# ################################################################################
-
-# sub verify_uniqueness_diff {
-#     my $warp_id_1 = shift;
-#     my $warp_id_2 = shift;
-#     my $date = shift;
-#     my $target = shift;
-
-#     my $db = init_gpc_db();
-#     $date =~ s/-//g;
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-
-#     my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
-#     my $data_ref = $db->selectall_arrayref( $sth );
-
-#     return($#{ $data_ref } + 1);
-# }
-
-# sub do_extra_processing {
-#     my $date = shift;
-#     my $target = shift;
-#     my $filter = shift;
-#     my $pretend = shift;
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
-
-#     if ($target eq 'OSS') {
-# 	my $db = init_gpc_db();
-
-# 	my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-# 	$obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
-# 	print STDERR "$obj_sth\n";
-# 	my $object_ref = $db->selectall_arrayref( $obj_sth );
-
-# 	foreach my $object_row (@{ $object_ref }) {
-# 	    my $this_object = shift @{ $object_row };
-# 	    my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-# 	    $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
-# 	    $input_sth .= " ORDER BY dateobs ";
-
-# 	    my $warps = $db->selectall_arrayref( $input_sth );
+#
+# Stacking
+################################################################################
+
+
+sub construct_stacktool_cmd {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$stacktool";
+#    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
+    $cmd .= " -set_label $label -select_label $label ";
+    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
+    $cmd .= " -select_filter $filter -set_data_group $data_group ";
+    $cmd .= " -min_num 4";
+    $cmd .= " -select_good_frac_min 0.05";
+    $cmd .= " $select ";
+    if ($debug == 1) {
+        $cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}
+
+sub verify_uniqueness_stack {
+    my $skycell = shift;
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_stack_queue {
+    my $date = shift;
+    my $observing_state = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $where = " label = '$label' AND data_group = '$data_group' ";
+
+    my $where_possibly_faulted = $where . " AND ( " ;
+    foreach my $acceptable_quality (@unrecoverable_quality) {
+	$where_possibly_faulted .= " quality = $acceptable_quality OR ";
+    }
+    $where_possibly_faulted .= " 0 )";
+
+    my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
+    my $cam_sth  = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted ";
+    my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
+
+    my $chip_ref = $db->selectall_arrayref( $chip_sth );
+    my $cam_ref  = $db->selectall_arrayref( $cam_sth );
+    my $warp_ref = $db->selectall_arrayref( $warp_sth );
+
+    # check that we will be able to queue them up
+    my $command = construct_stacktool_cmd($date,$target,$filter) . ' -pretend ';
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stacktool: $error_code", $#{ $chip_ref },$#{ $warp_ref },$date, $PS_EXIT_SYS_ERROR);
+    }
+
+    my $Nalready = 0;
+
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+    foreach my $entry (@input_exposures) {
+        my ($warp_tess_id,$skycell, @trash) = split /\s+/, $entry;
+        @trash = verify_uniqueness_stack($skycell,$date,$target,$filter);
+        $Nalready += $trash[0];
+    }
+    
+    my $minimum_required_warp;
+    if ($observing_state eq 'OBSERVING') {
+	$minimum_required_warp = $science_config{$target}{MIN_STACK};
+    }
+    unless (defined($minimum_required_warp)) {
+	$minimum_required_warp = 0;
+    }
+
+    return($#input_exposures  + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $minimum_required_warp, $Nalready);
+}
+
+sub stack_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+
+    my $command = construct_stacktool_cmd($date,$target,$filter);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $command, verbose => $verbose );
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    
+    return(0);
+}
+
+sub execute_stacks {
+    my $date = shift;
+    my $observing_state = shift;
+    my $pretend = shift;
+
+    foreach my $target (sort (keys %science_config)) {
+	if ($science_config{$target}{STACKABLE} == 1) {
+	    foreach my $filter (@filter_list) {
+                my ($Nexposures,$NprocChips,$NprocWarps,$NrequiredWarps,$Nalready) = pre_stack_queue($date,$observing_state,$target,$filter);
+                if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
+		    if ($debug == 1) {
+			print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
+		    }
+                    $metadata_out{nsStackState} = 'FORCETOWARP';
+                    next;
+                }
+                if ($Nexposures == 0) {
+		    if ($debug == 1) {
+			print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
+		    }
+                    next;
+                }
+		if ($NprocWarps < $NrequiredWarps) {
+		    if ($debug == 1) {
+			print STDERR "execute_stacks: Target $target in filter $filter on $date has too few warps to begin stacking. ($NprocWarps $NrequiredWarps)\n";
+		    }
+		    next;
+		}
+                if ($Nalready != 0) {
+		    if ($debug == 1) {
+			print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
+		    }
+		    unless ($metadata_out{nsStackState} eq 'FORCETOWARP') {
+			$metadata_out{nsStackState} = 'STACKING';
+		    }
+                    next;
+                }
+                unless (($metadata_out{nsStackState} eq 'FORCETOWARP')||($metadata_out{nsStackState} eq 'STACKING')) {
+                    $metadata_out{nsStackState} = 'QUEUESTACKS';
+                }
+                unless(defined($pretend)) {
+		    if ($debug == 1) {
+			print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
+		    }
+                    stack_queue($date,$target,$filter);
+                }
+		if (defined($pretend)) {
+		    add_to_macro_list('check_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter);
+		}
+		else {
+		    add_to_macro_list('queue_stacks',$science_config{$target}{STACKABLE},$date,$target,$filter);
+		}
+            }
+	}
+	else {
+	    if ($debug == 1) {
+		print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+	    }
+        }
+    }
+}
+
+sub post_stack_queue {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+    # check warp stage == chip stage
+    my $db = init_gpc_db();
+
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $where = " label = '$label' AND data_group = '$data_group' ";
+
+    my $stack_full_sth = "SELECT * from stackRun where $where ";
+    my $stack_done_sth = "SELECT * from stackRun LEFT JOIN stackSumSkyfile USING(stack_id) WHERE $where ";
+    $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) ";
+    if ($debug == 1) {
+	print STDERR "post_stack_queue: database queries:\n";
+	print STDERR "$stack_full_sth\n";
+	print STDERR "$stack_done_sth\n";
+    }
+    my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth );
+    my $stack_done_ref = $db->selectall_arrayref( $stack_done_sth );
+
+    return($#{ $stack_full_ref } + 1, $#{ $stack_done_ref } + 1);
+}
+
+sub confirm_stacks {
+    my $date = shift;
+    my $pretend = shift;
+    
+    foreach my $target (sort (keys %science_config)) {
+        if ($science_config{$target}{STACKABLE} == 1) {
+	    my ($Nstacks, $Nattempted) = post_stack_queue($date,$target);
+	    if ($debug == 1) {
+		print STDERR "confirm_stacks: Target $target on $date has $Nattempted attempts on $Nstacks.\n";
+	    }
+	    if ($Nstacks != $Nattempted) {
+		if ($debug == 1) {
+		    print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n";
+		}
+		if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') {
+		    $metadata_out{nsStackState} = 'STACKING';
+		}
+		next;
+	    }
+	    if ($Nstacks == 0) {
+		if ($debug == 1) {
+		    print STDERR "confirm_stacks: Target $target on $date has no stacks. Skipping.\n";
+		}
+		next;
+	    }
+	    if ($metadata_out{nsStackState} eq 'CONFIRM_STACKING') {
+		if (defined($pretend)) {
+		    add_to_macro_list('check_confirm_stacks',$science_config{$target}{STACKABLE},$date,$target);
+		}
+		else {
+		    add_to_macro_list('confirm_stacks',$science_config{$target}{STACKABLE},$date,$target);
+		}
+	    }
+	}
+    }
+}	    
+
+#
+# Extra processing
+################################################################################
+
+sub verify_uniqueness_diff {
+    my $warp_id_1 = shift;
+    my $warp_id_2 = shift;
+    my $date = shift;
+    my $target = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_diff_queue {
+    my $date = shift;
+    my $observing_state = shift;
+    my $target = shift;
+    my $filter = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+    
+    my $db = init_gpc_db();
+
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $where = " label = '$label' AND data_group = '$data_group' ";
+
+    my $where_possibly_faulted = $where . " AND ( " ;
+    foreach my $acceptable_quality (@unrecoverable_quality) {
+	$where_possibly_faulted .= " quality = $acceptable_quality OR ";
+    }
+    $where_possibly_faulted .= " 0 )";
+
+    my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
+    my $cam_sth  = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted ";
+    my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
+
+    my $chip_ref = $db->selectall_arrayref( $chip_sth );
+    my $cam_ref  = $db->selectall_arrayref( $cam_sth );
+    my $warp_ref = $db->selectall_arrayref( $warp_sth );
+
+    return($#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2);
+}
+
+sub execute_diffs {
+    my $date = shift;
+    my $observing_state = shift;
+    my $pretend = shift;
+
+    foreach my $target (sort (keys %science_config)) {
+	if ($science_config{$target}{DIFFABLE} == 1) {
+	    foreach my $filter (@filter_list) {
+		my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
+		if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
+		    if ($debug == 1) {
+			print STDERR "execute_diffs: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
+		    }
+                    $metadata_out{nsDiffState} = 'FORCETOWARP';
+                    next;
+                }
+                if ($NprocChips == 0) {
+		    if ($debug == 1) {
+			print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n";
+		    }
+                    next;
+                }
+                unless (($metadata_out{nsDiffState} eq 'FORCETOWARP')||($metadata_out{nsDiffState} eq 'DIFFING')) {
+                    $metadata_out{nsDiffState} = 'QUEUEDIFFS';
+                }
+		if ($debug == 1) {
+		    print STDERR "execute_diffs: Target $target in filter $filter on $date has exposures and will be queued.\n";
+		}
+		diff_queue($date,$target,$filter,$pretend);
+
+		if (defined($pretend)) {
+		    add_to_macro_list('check_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter);
+		}
+		else {
+		    add_to_macro_list('queue_diffs',$science_config{$target}{DIFFABLE},$date,$target,$filter);
+		}
+            }
+	}
+	else {
+	    if ($debug == 1) {
+		print STDERR "execute_diffs: Target $target is not auto-diffable.\n";
+	    }
+        }
+    }
+}
+
+sub diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+    
+    foreach my $object_row (@{ $object_ref }) {
+	my $this_object = shift @{ $object_row };
+	my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+	$input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+	$input_sth .= " ORDER BY dateobs ";
+	
+	my $warps = $db->selectall_arrayref( $input_sth );
+	
+	if (($#{ $warps } + 1) % 2 != 0) {
+	    print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
+	    next;
+	}
+	
+	while ($#{ $warps } > -1) {
+	    my $input_warp = shift @{ $warps };
+	    my $template_warp = shift @{ $warps };
+	    my $input_exp_id = ${ $input_warp }[0];
+	    my $template_exp_id = ${ $template_warp }[0];
 	    
-# 	    if (($#{ $warps } + 1) % 2 != 0) {
-# 		print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
-# 		last;
-# 	    }
+	    my $input_warp_id = ${ $input_warp }[1];
+	    my $template_warp_id = ${ $template_warp }[1];
 	    
-# 	    while ($#{ $warps } > -1) {
-# 		my $input_warp = shift @{ $warps };
-# 		my $template_warp = shift @{ $warps };
-# 		my $input_exp_id = ${ $input_warp }[0];
-# 		my $template_exp_id = ${ $template_warp }[0];
-		
-# 		my $input_warp_id = ${ $input_warp }[1];
-# 		my $template_warp_id = ${ $template_warp }[1];
-
-# 		if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
-# 		    print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
-# 		    next;
-# 		}
-
-# 		my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
-# 		$cmd .= "-input_label $label  -template_label $label ";
-# 		$cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
-# 		$cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
-# 		$cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
-# #		$cmd .= " -pretend ";
-# 		if (defined($pretend)) {
-# 		    $cmd .= ' -pretend ';
-# 		}
-# 		if ($debug == 1) {
-# 		    $cmd .= ' -pretend ';
-# 		}
-# 		print STDERR "EXTRA_PROCESSING: $cmd\n";
-# 		if (($debug == 0)&&(!defined($pretend))) {
-# 		    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-# 			run ( command => $cmd, verbose => $verbose );
-# 		    unless ($success) {
-# 			$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-# 			&my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-# 		    }
-# 		}
-# 	    }
-# 	}
-#     }
-# }
-
+	    if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+		print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
+		next;
+	    }
+	    
+	    my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+	    $cmd .= "-input_label $label  -template_label $label ";
+	    $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+	    $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+	    $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#		$cmd .= " -pretend ";
+	    if (defined($pretend)) {
+		$cmd .= ' -pretend ';
+	    }
+	    if ($debug == 1) {
+		$cmd .= ' -pretend ';
+		print STDERR "diff_queue: $cmd\n";
+	    }
+	    
+	    if (($debug == 0)&&(!defined($pretend))) {
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run ( command => $cmd, verbose => $verbose );
+		unless ($success) {
+		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		}
+	    }
+	}
+    }
+}
 	    
 
@@ -1302,76 +1386,76 @@
 ################################################################################
 
-# sub construct_cleantool_args {
-#     my $date = shift;
-#     my $target = shift;
-#     my $mode = shift;
-
-#     my $command = $clean_commands{$mode};
-#     my $retention_time;
-#     if (exists($cleanmods_list{$target}{$mode})) {
-# 	$retention_time = $cleanmods_list{$target}{$mode};
-#     }
-#     else {
-# 	$retention_time = $clean_retention{$mode};
-#     }
-#     if ($retention_time <= 0) {
-# 	return("no clean","true");
-#     }
-
-#     my ($year,$month,$day) = split /-/,$date;
-#     my $dt = DateTime->new(year => $year, month => $month, day => $day,
-#                                hour => 0, minute => 0, second => 0, nanosecond => 0,
-#                                time_zone => 'Pacific/Honolulu');
+sub construct_cleantool_args {
+    my $date = shift;
+    my $target = shift;
+    my $mode = shift;
+
+    my $command = $clean_config{$mode}{COMMAND};
+    my $retention_time;
+    if (exists($science_config{$target}{$mode})) {
+	$retention_time = $science_config{$target}{$mode};
+    }
+    else {
+	$retention_time = $clean_config{$mode}{RETENTION_TIME};
+    }
+    if ($retention_time <= 0) {
+	return("no clean","true");
+    }
+
+    my ($year,$month,$day) = split /-/,$date;
+    my $dt = DateTime->new(year => $year, month => $month, day => $day,
+			   hour => 0, minute => 0, second => 0, nanosecond => 0,
+			   time_zone => 'Pacific/Honolulu');
 	
-#     $dt->subtract(days => $retention_time);
-#     my $cleaning_date = $dt->ymd;
-
-#     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
-    
-#     $command =~ s/\@DBNAME\@/$dbname/;
-#     $command =~ s/\@LABEL\@/$label/;
-#     $command =~ s/\@WORKDIR\@/$workdir/;
-#     $command =~ s/\@OBS_MODE\@/$obs_mode/;
-#     $command =~ s/\@OBJECT\@/$object/;
-#     $command =~ s/\@TESS_ID\@/$tess_id/;
-#     $command =~ s/\@DIST_GROUP\@/$dist_group/;
-#     $command =~ s/\@DATA_GROUP\@/$data_group/;
-#     $command =~ s/\@REDUCTION\@/$reduction/;
-#     $command =~ s/\@CURRENT_DATE\@/$cleaning_date/;
-    
-#     if ($debug == 1) {
-#         $command .= ' -pretend ';
-#     }
-#     return($cleaning_date,$command);
-# }
-
-# sub execute_cleans {
-#     my $date = shift;
-#     my $pretend = shift;
-
-#     foreach my $mode (sort (keys %clean_config)) {
-# 	foreach my $target (sort (keys %science_config)) {
-# 	    if (exists($noclean_list{$target})) {
-# 		next;
-# 	    }
-# 	    my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
-# 	    if ($cleaning_date eq 'no clean') {
-# 		next;
-# 	    }		
-# 	    print STDERR "$command\n";
-# 	    if (!(defined($pretend) || $debug == 1)) {
-# #           print STDERR "BEAR IS DRIVING!?\n";
-# 		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-# 		    run ( command => $command, verbose => $verbose );
-# 		unless ($success) {
-# 		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-# 		    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-# 		}
-# 		add_to_macro_list('clean_old',1,$date,$target,$mode);
-# 	    }
-# 	}
-#     }
-#     return(0);
-# }
+    $dt->subtract(days => $retention_time);
+    my $cleaning_date = $dt->ymd;
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
+    
+    $command =~ s/\@DBNAME\@/$dbname/;
+    $command =~ s/\@LABEL\@/$label/;
+    $command =~ s/\@WORKDIR\@/$workdir/;
+    $command =~ s/\@OBSMODE\@/$obs_mode/;
+    $command =~ s/\@OBJECT\@/$object/;
+    $command =~ s/\@TESS_ID\@/$tess_id/;
+    $command =~ s/\@DIST_GROUP\@/$dist_group/;
+    $command =~ s/\@DATA_GROUP\@/$data_group/;
+    $command =~ s/\@REDUCTION\@/$reduction/;
+    $command =~ s/\@CURRENT_DATE\@/$cleaning_date/;
+    
+    if ($debug == 1) {
+        $command .= ' -pretend ';
+    }
+    return($cleaning_date,$command);
+}
+
+sub execute_cleans {
+    my $date = shift;
+    my $pretend = shift;
+
+    foreach my $mode (sort (keys %clean_config)) {
+	foreach my $target (sort (keys %science_config)) {
+	    if (exists($science_config{$target}{NOCLEAN})) {
+		next;
+	    }
+	    my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
+	    if ($cleaning_date eq 'no clean') {
+		next;
+	    }		
+	    print STDERR "$command\n";
+	    if (!(defined($pretend) || $debug == 1)) {
+#           print STDERR "BEAR IS DRIVING!?\n";
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run ( command => $command, verbose => $verbose );
+		unless ($success) {
+		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		}
+		add_to_macro_list('clean_old',1,$date,$target,$mode);
+	    }
+	}
+    }
+    return(0);
+}
 
 #
@@ -1426,5 +1510,5 @@
     my $label      = "${target}.nightlyscience";
     my $workdir    = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}";
-    my $obs_mode   = $science_config{$target}{OBS_MODE};
+    my $obs_mode   = $science_config{$target}{OBSMODE};
     my $object     = $science_config{$target}{OBJECT};
     my $comment    = $science_config{$target}{COMMENT};
@@ -1442,7 +1526,7 @@
     my $trunc_date = $date; $trunc_date =~ s/-//g;
 
-    my $exp_type   = $detrend_config{$target}{EXP_TYPE};
-    my $det_type   = $detrend_config{$target}{DET_TYPE};
-    my $ref_det_id = $detrend_config{$target}{REF_DET_ID};
+    my $exp_type   = $detrend_config{$target}{EXPTYPE};
+    my $det_type   = $detrend_config{$target}{DETTYPE};
+    my $ref_det_id = $detrend_config{$target}{REF_ID};
     my $ref_iter   = $detrend_config{$target}{REF_ITER};
     my $maxN       = $detrend_config{$target}{MAX_EXP};
@@ -1455,5 +1539,8 @@
     else {
 	$internal_filter = '';
-    }    
+    }
+    unless (defined($maxN)) {
+	$maxN = 0;
+    }
     my $lc_type = lc($exp_type);
     my $label = "${lc_type}${internal_filter}.$trunc_date";
@@ -1484,5 +1571,5 @@
 	    $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/;
 	    $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/;
-	    $metadata_out{"ns${N}Macro"} =~ s/\@OBS_MODE\@/$obs_mode/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@OBSMODE\@/$obs_mode/;
 	    $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/;
 	    $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/;