Index: branches/czw_branch/20101203/ippScripts/Build.PL
===================================================================
--- branches/czw_branch/20101203/ippScripts/Build.PL	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/Build.PL	(revision 30586)
@@ -58,4 +58,5 @@
         scripts/magic_tree.pl
         scripts/magic_process.pl
+        scripts/magic_cleanup.pl
         scripts/magic_destreak.pl
         scripts/magic_destreak_revert.pl
Index: branches/czw_branch/20101203/ippScripts/scripts/addstar_run.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/addstar_run.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/addstar_run.pl	(revision 30586)
@@ -71,6 +71,6 @@
     defined $camera;
 if ($minidvodb && !defined($minidvodb_group)) {
-		my_die( "missing minidvodb_group", $add_id, 3 );
-	    }
+                my_die( "missing minidvodb_group", $add_id, 3 );
+            }
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
@@ -113,38 +113,39 @@
 
 if (defined $dvodbReal) {
-	if ($minidvodb) {
-	    
-	    if (!defined($minidvodb_name)) {
-		#take the active one, if it's not defined on the command line
-		#reverts would have this already set, for example.
-		my $command = "addtool -listminidvodbrun ";
-		$command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
-		$command .= " -state 'active' -limit 1";
-		$command .= " -dbname $dbname" if defined $dbname;
-		print $command;
-		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		    run(command => $command, verbose => $verbose);
-		&my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
-		my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-		    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
-
-		my $components = parse_md_list($metadata) or
-		    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
-		my $comp = $$components[0];
-		$minidvodb_path = $comp->{minidvodb_path};
-		$minidvodb_name = $comp->{minidvodb_name};
-	
-		if (!defined($minidvodb_path)) {
-		    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
-		} 
-		if (!defined($minidvodb_name)) {
-		    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
-		}
-	    }
-	    # tack on the minidvodb part to the db.
-#	    $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
-#we don't need this now that I fixed the paths	   
-	    
-	}
+        if ($minidvodb) {
+
+            #if (!defined($minidvodb_name)) {
+                #take the active one, if it's not defined on the command line
+                #reverts would have this already set, for example.
+                my $command = "addtool -listminidvodbrun ";
+                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
+                $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+                $command .= " -state 'active' -limit 1";
+                $command .= " -dbname $dbname" if defined $dbname;
+                print $command;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
+
+                my $components = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+                my $comp = $$components[0];
+                $minidvodb_path = $comp->{minidvodb_path};
+                $minidvodb_name = $comp->{minidvodb_name};
+
+                if (!defined($minidvodb_path)) {
+                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
+                }
+                if (!defined($minidvodb_name)) {
+                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
+                }
+            #}
+            # tack on the minidvodb part to the db.
+#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
+#we don't need this now that I fixed the paths
+
+        }
         else {
 
@@ -153,5 +154,5 @@
 
 unless ($no_op) {
-   	    print $dvodbReal;
+            print $dvodbReal;
 
         ## addstar can either save the full set of detections, or just
@@ -222,8 +223,8 @@
         $command .= " -addprocessedexp";
         $command .= " -fault $exit_code";
-	$command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
+        $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
-	$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_advancerun.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_advancerun.pl	(revision 30586)
@@ -23,5 +23,4 @@
 use PS::IPP::Config 1.01 qw( :standard );
 
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -29,9 +28,10 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $outdir, $clean);
+my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
-           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'dist_id=s'      => \$dist_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -53,4 +53,6 @@
     defined $stage_id and
     defined $outdir;
+
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -134,9 +136,46 @@
 }
 
-# XXX should we create a file rule for this?
-my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $exportarg -outfile $dbinfo_file";
+
+# work around the fact that $ipprc->file_create does not actually create a file on disk
+# unless the scheme is nebulous
+sub create_file {
+    my $rule = shift;
+    my $path_base = shift;
+    my $ref = shift;
+
+    my $file;
+
+    my $error;
+    $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error)
+        or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR);
+
+    my $scheme = file_scheme($file);
+    $scheme = "" if !$scheme;
+
+    my $resolved;
+    if ($scheme) {
+        $ipprc->file_create($file) 
+            or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        $resolved = $ipprc->file_resolve($file) 
+            or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        if ($scheme eq 'neb') {
+            &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved);
+    }
+    } else {
+        $resolved = $file;
+    }
+    $$ref = $resolved;
+
+    return $file
+}
+
+my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id";
+my $resolved;
+my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved);
+
+{
+    my $command = "$tool_cmd $exportarg -outfile $resolved";
     $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
@@ -149,5 +188,8 @@
     }
 }
-my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+
+my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id";
+my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved);
+
 {
     my $command = "$tool_cmd $list_mode";
@@ -168,6 +210,6 @@
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
-    open MANIFEST, ">$dirinfo" or
-        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    open MANIFEST, ">$resolved" or
+        &my_die("Unable to open dirinfo file $resolved",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
     my $destdir;
@@ -197,5 +239,4 @@
         &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
         my $component_dir = find_componentdir($destdir, $path);
-#        print MANIFEST "$component METADATA\n";
         print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
     }
@@ -217,5 +258,4 @@
     }
 }
-
 
 exit 0;
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl	(revision 30586)
@@ -16,5 +16,5 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename );
 use Digest::MD5::File qw( file_md5_hex );
@@ -71,5 +71,5 @@
 # Parse the command-line arguments
 my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean);
-my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
+my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -88,5 +88,5 @@
            'magicked'       => \$magicked,   # magicked state for this component
            'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
-           'outdir=s'       => \$outdir,     # "directory" for outputs
+           'outroot=s'      => \$outroot,    # outroot
            'prefix=s'       => \$prefix,     # "prefix" to apply to filenames
            'clean'          => \$clean,      # create clean distribution
@@ -99,5 +99,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir",
+pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot",
            -exitval => 3) unless
     defined $camera and
@@ -106,5 +106,5 @@
     defined $component and
     defined $path_base and
-    defined $outdir;
+    defined $outroot;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -116,12 +116,5 @@
 $ipprc->define_camera($camera);
 
-# create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outdir, "neb://") != 0) {
-    if (! -e $outdir ) {
-        my $code = system "mkdir -p $outdir";
-        &my_die("cannot create output directory $outdir", $component,
-                $code >> 8) if $code;
-    }
-}
+$ipprc->outroot_prepare($outroot);
 
 # Get the list of data products for this component
@@ -138,15 +131,9 @@
 # set up directory for temporary files
 
-my $tmpdir  = "$outdir/tmpdir.$component.$$";
-if (-e $tmpdir) {
-    if (-d $tmpdir) {
-        my $rc = system "rm -r $tmpdir";
-        &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
-    } else {
-        unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
-mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component,
-                       $PS_EXIT_UNKNOWN_ERROR);
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+$temproot = "/tmp" if !defined $temproot;
+&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot;
+
+my $tmpdir  = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps);
 
 #
@@ -349,9 +336,36 @@
 if ($num_files) {
     # create the tarfile
-    # XXX TODO: create a file rule for the tar file name
-    my $tbase = basename($path_base);
-    $tbase .= ".$component" if $component;
-    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
-    my $tarfile = "$outdir/$file_name";
+#    my $tbase = basename($path_base);
+#    $tbase .= ".$component" if $component;
+#    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
+    my $error;
+    my $output_tarfile;
+    my $rule;
+    if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') {
+        $rule = "DIST.OUTPUT.CHIP.BUNDLE";
+    } else {
+        $rule = "DIST.OUTPUT.BUNDLE";
+    }
+    $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error)
+            or &my_die("Failed to prepare output tarfile: $error", $component, $error);
+    $file_name = basename($output_tarfile);
+
+
+    my $scheme = file_scheme($output_tarfile);
+
+    my $tarfile;
+    if ($scheme) {
+        $ipprc->file_create($output_tarfile) 
+            or &my_die("Failed to create $output_tarfile: $error", $component, $error);
+        $tarfile = $ipprc->file_resolve($output_tarfile);
+
+        if ($scheme eq 'neb') {
+            &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR)
+                unless ($tarfile and -e $tarfile);
+        }
+    } else {
+        # no scheme the filename and the resolved filename are the same
+        $tarfile = $output_tarfile;
+    }
 
     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_component.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_component.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_component.pl	(revision 30586)
@@ -21,8 +21,5 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-
 use PS::IPP::Config 1.01 qw( :standard );
-
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -77,5 +74,10 @@
     defined $outdir;
 
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 $ipprc->redirect_output($logfile) if $logfile;
+
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+
+$temproot = "/tmp" if !defined $temproot;
 
 if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
@@ -85,6 +87,9 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
+my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
 close $rf;
+
+my $basename = basename($path_base);
+my $outroot = "$outdir/$basename";
 
 my ($file_name, $bytes, $md5sum);
@@ -92,5 +97,5 @@
     my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --results_file $rf_name";
-    $command .= " --component $component --path_base $path_base --outdir $outdir";
+    $command .= " --component $component --path_base $path_base --outroot $outroot";
     $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     $command .= " --state $run_state" if defined $run_state;
@@ -136,4 +141,5 @@
     &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
 }
+
 
 {
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_defineruns.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_defineruns.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_defineruns.pl	(revision 30586)
@@ -42,5 +42,5 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $dist_root, $no_magic);
+my ($stage, $stage_limit, $dist_root, $workdir, $no_magic);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -49,6 +49,7 @@
            'stage=s'        => \$stage,      # stage to queue
            'label=s'        => \@labels,     # labels
+           'workdir=s'      => \$workdir,     # workdir
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
-           'dist_root=s'    => \$dist_root,  # root of distribution work area
+#           'dist_root=s'    => \$dist_root,  # root of distribution work area
            'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
            'dbname=s'       => \$dbname,     # Database name
@@ -66,15 +67,19 @@
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (!$dist_root) {
-    $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
-    &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+if (!$workdir) {
+    print "workdir not supplied will use DISTRIBUTION_ROOT\n";
+    # old method where we set workdir based on a config file
+    if (!$dist_root) {
+        $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
+        &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+    }
+
+    my ($day, $month, $year) = (gmtime)[3,4,5];
+    my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+
+    $workdir = $dist_root . "/$datestr";
+
+    print "workdir is $workdir\n";
 }
-
-my ($day, $month, $year) = (gmtime)[3,4,5];
-my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
-
-my $workdir = "$dist_root/$datestr";
-
-print "workdir is $workdir\n";
 
 # if stage is not supplied as an argument, loop over all stages
@@ -90,5 +95,5 @@
         my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label";
         $command .= " -no_magic" if $no_magic;
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_make_fileset.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_make_fileset.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_make_fileset.pl	(revision 30586)
@@ -96,5 +96,5 @@
 # make sure that the database info file for this run exists
 my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
-if (! -e "$dbinfo_file" ) {
+if (! $ipprc->file_exists($dbinfo_file) ) {
     &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -103,5 +103,5 @@
 # make sure that the dirinfo file for this run exists
 my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
-if (! -e "$dirinfo_file" ) {
+if (!$ipprc->file_exists($dirinfo_file)) {
     &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -277,9 +277,13 @@
     $command .= " -dbname $dbname" if $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        print STDERR "Unable to perform $command error_code: $error_code\n";
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR "Unable to perform $command error_code: $error_code\n";
+        }
+    } else {
+        print STDERR "skipping $command\n";
     }
     exit $fault;
Index: branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl	(revision 30586)
@@ -74,9 +74,9 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 }
 
 my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to parse metadata config doc", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 
 my $burntoolStateGood = 999;
@@ -87,5 +87,5 @@
 }
 if ($burntoolStateGood == 999) {
-    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
+    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
 }
 my $outState = -1 * abs($burntoolStateGood);
@@ -178,5 +178,5 @@
         my $exp_id = shift;
         my $class_id = shift;
-        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
+        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state 0 -set_state pending_burntool",1);
     }
     printf STDERR "$message\n";
Index: branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl	(revision 30586)
@@ -167,7 +167,7 @@
 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
-            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
+            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
@@ -429,7 +429,7 @@
 
                 # delete the temporary image datafiles
-                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
-                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
-                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
+                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
+                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
                 if ($mode eq "goto_purged") {
@@ -706,11 +706,11 @@
             my @files = ();
             # delete the temporary image datafiles
-            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
-
-            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
-            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
+            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
+
+            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id, 1);
+            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id, 1);
 
             addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
@@ -2048,8 +2048,15 @@
     my $path_base  = shift; # base filename
     my $class_id   = shift; # class_id, if needed
+    my $recovery   = shift; # is there is a recovery file to clean?
 
     my $file = $ipprc->filename($filerule, $path_base, $class_id);
 
     push @$files, $file;
+    
+    if ($recovery) {
+        # need to clean up the recovery file (the pixels censored by streaksremove)
+        $file = $ipprc->recovery_filename($file);
+        push @$files, $file;
+    }
     return 1;
 }
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_cleanup.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_cleanup.pl	(revision 30586)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_cleanup.pl	(revision 30586)
@@ -0,0 +1,158 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+my $date = `date`;
+print "\n\n";
+print "Starting script $0 on $host at $date\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename dirname );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
+use DBI;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $magictool   = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($magic_id, $exp_id, $outroot, $camera);
+my ($dbname, $verbose, $no_update, $no_op, $logfile);
+
+GetOptions(
+           'magic_id=s'     => \$magic_id,   # Magic run identifier
+           'outroot=s'      => \$outroot,    # directory to clean up
+           'camera=s'       => \$camera,    # directory to clean up
+           'logfile=s'      => \$logfile,
+
+           'exp_id=s'       => \$exp_id,     # exposure id # XXX: do we need this?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database?
+           'no-op'          => \$no_op,      # Don't do any operations?
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --magic_id --exp_id --outroot --camera",
+           -exitval => 3) unless
+    defined $magic_id and
+    defined $exp_id and
+    defined $outroot and 
+    defined $camera;
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->outroot_prepare($outroot);
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+my $parentdir = dirname($outroot);
+if (-e $parentdir ) {
+    my @extensions = ('.list', '.streakMap', '.streaks', '.clusters', '_hough.fits');
+    foreach my $ext (@extensions) {
+        my $n = 0;
+        my $path = $outroot . '*' . $ext;
+        my @files = glob($path);
+        foreach my $file (@files) {
+            if (-e $file) {
+                $n++;
+                unless ($no_op) {
+                    # remove this print
+                    # print "unlinking $file\n";
+                    unlink $file or my_die ("unable to delete $file", $magic_id, $PS_EXIT_SYS_ERROR);
+                } else {
+                    print "Skipping unlink $file\n";
+                }
+            }
+        }
+        print "Deleted $n $ext files\n" unless $no_op;
+    }
+    # remove the verify directory
+    my $verify_dir = $parentdir . "/$exp_id.mgc.$magic_id.verify";
+    if (-e $verify_dir ) {
+        my $rc = system "rm -r $verify_dir";
+        my_die("failed to rm $verify_dir",  $magic_id, $rc >> 8) if $rc;
+        print "deleted $verify_dir\n";
+    }
+} else {
+    # parent directory doesn't "exist" either, check that mountpoint exists
+    # if it doesn't assume nfs error and fault
+    # we expect at least /data/volume
+    my @dirs = split "/", $parentdir;
+    # so the following should be true
+    # $dirs[0] eq ""
+    # $dirs[1] eq 'data'
+    # $dirs[2] eq the volume: for example ipp053.0
+    if ((scalar(@dirs) < 3) or ($dirs[0] ne "") or ($dirs[1] ne 'data')) {
+        my_die("unexpected parent of outroot found: '$parentdir'", $magic_id, $PS_EXIT_UNKNOWN_ERROR);
+    } else { 
+        # re-form mountpoint
+        my $mountpoint = "/$dirs[1]/$dirs[2]";
+        # check if it is readable
+        if (! -e $mountpoint or ! -r $mountpoint ) {
+            my_die("mount point for outroot not found or not readable: '$mountpoint'", $magic_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    print "Workdir $outroot not found. Setting magicRun.workdir_state to cleaned.\n";
+    # fall through to update workdir_state
+}
+            
+my $command = "$magictool -setworkdirstate -magic_id $magic_id -set_workdir_state cleaned";
+$command   .= " -dbname $dbname" if defined $dbname;
+unless ($no_update) {
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        carp("failed to update database for $magic_id");
+    }
+} else {
+    print "Skipping command: $command\n";
+}
+exit 0;
+    
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $magic_id = shift;    # Magic DS identifier
+    my $exit_code = shift;      # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
+
+    my $command = "$magictool -setworkdirstate -set_workdir_state error_cleaned";
+    $command   .= " -magic_id $magic_id";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_id");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
+    carp($msg);
+    exit $exit_code;
+}
+
+__END__
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl	(revision 30586)
@@ -65,5 +65,5 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recoveryroot=s' => \$recoveryroot,# "prefix" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'magicked=s'     => \$magicked,   # magicked state of the run
@@ -190,13 +190,16 @@
 my $recovery_path_base;
 if ($recoveryroot) {
-    if (!inNebulous($recoveryroot)) {
-        if (! -e $recoveryroot ) {
-            my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
-                    $code >> 8) if $code;
-        }
-    }
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
+    # recoveryroot is a path to prepend to the basenames of the input files
+    if (inNebulous($recoveryroot)) {
+        # if recoveryroot is a nebulous path we ignore the actual path and put the files in the
+        # the same "directory" as the input files
+        $recoveryroot = "$dirname/REC_";
+        $recovery_path_base = $recoveryroot . $basename;
+    } else {
+        # otherwise we put the files in recoveryroot.
+        # Regardless, we prefix the basename with 'REC_'
+        $ipprc->outroot_prepare($recoveryroot);
+        $recovery_path_base = "$recoveryroot/REC_$basename";
+    }
 }
 
@@ -601,4 +604,6 @@
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
+    $command .= " -backup_path_base $backup_path_base" if $backup_path_base;
+    $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl	(revision 30586)
@@ -233,5 +233,7 @@
                 $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
                 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
             if ($recovery_path_base) {
@@ -251,5 +253,7 @@
                 $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
                 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
             if ($recovery_path_base) {
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl	(revision 30586)
@@ -42,5 +42,5 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $workdir);
+my ($stage, $stage_limit, $workdir, $recoveryroot);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -51,4 +51,5 @@
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
            'workdir=s'      => \$workdir,    # output destination
+           'recoveryroot=s' => \$recoveryroot, # recovery pixels destination
            'dbname=s'       => \$dbname,     # Database name
            'verbose'        => \$verbose,    # Print stuff?
@@ -78,4 +79,5 @@
     foreach my $label (@labels) {
         my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
+        $command .= " -recoveryroot $recoveryroot" if $recoveryroot;
         $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl	(revision 30586)
@@ -39,5 +39,5 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked, $run_state);
-my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
+my ($outroot, $recovery_path_base, $replace, $release, $bytes, $md5sum);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -53,5 +53,5 @@
            'cam_reduction=s'=> \$cam_reduction,  # reduction class of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
-           'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
+           'recovery_path_base=s' => \$recovery_path_base,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
            'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
@@ -130,6 +130,6 @@
 
 # default value is "NULL" do not use it
-if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
-    $recoveryroot = undef;
+if (defined($recovery_path_base) and ($recovery_path_base eq "NULL")) {
+    $recovery_path_base = undef;
 }
 
@@ -149,13 +149,8 @@
 }
 
-my $recovery_path_base;
-if ($recoveryroot) {
-    # note: trailing / is necessary
-    $recovery_path_base = "$recoveryroot/$basename/";
-}
-
 
 my ($image, $mask, $ch_mask, $weight, $sources, $astrom);
 my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
+my ($rimage, $rmask, $rch_mask, $rweight);
 
 if ($stage eq "raw") {
@@ -214,4 +209,10 @@
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
+
+    $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
+    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
+    $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
+    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
+    $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
@@ -226,4 +227,7 @@
     $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
+    $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
+    $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
+    $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
 } elsif ($stage eq "diff") {
     my $name = "PPSUB.OUTPUT";
@@ -236,7 +240,15 @@
     $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+    }
 }
 
 revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+if ($recovery_path_base) {
+    delete_recovery_files($rimage, $rmask, $rweight, $rch_mask);
+}
 
 if ($stage eq "diff" and $bothways) {
@@ -251,4 +263,10 @@
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename($name, $recovery_path_base);
+        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
+        delete_recovery_files($rimage, $rmask, $rweight);
+    }
 }
 
@@ -426,9 +444,13 @@
 
 
-sub file_check
-{
-    my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
-        $ipprc->file_exists($file);
+sub delete_recovery_files
+{
+    foreach my $file (@_) {
+        if ($ipprc->file_exists($file)) {
+            if (!$ipprc->file_delete($file)) {
+                &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+            }
+        }
+    }
 }
 
Index: branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl	(revision 30586)
@@ -44,5 +44,5 @@
 my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
 my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
-my ( $registration_status, $burntool_status, $observing_status);
+my ( $registration_status, $burntool_status, $observing_status, $old_date);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
@@ -66,4 +66,5 @@
     'registraion_status'   => \$registration_status,
     'burntool_status'      => \$burntool_status,
+    'old_date=s'           => \$old_date,
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
@@ -134,5 +135,5 @@
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
+    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $date, $PS_EXIT_SYS_ERROR);
 }
 
@@ -278,4 +279,5 @@
 
 $metadata_out{nsObservingState} = &get_observing_state($date);
+$metadata_out{nsRegistrationState} = &get_registration_state($date);
 #
 # Mode selection
@@ -396,5 +398,5 @@
     exit(0);
 }
-exit(10);
+
 if (defined($check_confirm_stacks) || defined($test_mode)) {
     $metadata_out{nsStackState} = 'CONFIRM_STACKING';
@@ -409,5 +411,5 @@
     exit(0);
 }
-
+exit(10);
 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
     $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
@@ -1097,4 +1099,15 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+	$is_registering = 0;
+    }
+    else {
+	$is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
 	if ($science_config{$target}{STACKABLE} == 1) {
@@ -1105,4 +1118,5 @@
 			print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
 		    }
+		    $is_processing = 1;
                     $metadata_out{nsStackState} = 'FORCETOWARP';
                     next;
@@ -1120,5 +1134,7 @@
 		    next;
 		}
+		$Npotential++;
                 if ($Nalready != 0) {
+		    $Nqueued++;
 		    if ($debug == 1) {
 			print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
@@ -1137,4 +1153,5 @@
 		    }
                     stack_queue($date,$target,$filter);
+		    $Nqueued ++;
                 }
 		if (defined($pretend)) {
@@ -1152,4 +1169,9 @@
         }
     }
+    $metadata_out{nsStackPotential} = $Npotential;
+    $metadata_out{nsStackQueued}    = $Nqueued;
+    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)&&($is_registering == 0)) {
+	$metadata_out{nsStackState} = 'FINISHED_STACKS';
+    }	
 }
 
@@ -1231,8 +1253,15 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     my $data_ref = $db->selectall_arrayref( $sth );
-
-    return($#{ $data_ref } + 1);
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
 }
 
@@ -1272,7 +1301,19 @@
     my $pretend = shift;
 
+    my $Npotential = 0;
+    my $Nnoexp     = 0;
+    my $is_processing = 0;
+    my $is_registering;
+    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
+	$is_registering = 0;
+    }
+    else {
+	$is_registering = 1;
+    }
+
     foreach my $target (sort (keys %science_config)) {
 	if ($science_config{$target}{DIFFABLE} == 1) {
 	    foreach my $filter (@filter_list) {
+		$Npotential ++;
 		my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
 		if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
@@ -1280,8 +1321,8 @@
 			print STDERR "execute_diffs: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
 		    }
-                    $metadata_out{nsDiffState} = 'FORCETOWARP';
-                    next;
-                }
+		    $is_processing = 1;
+		}
                 if ($NprocChips == 0) {
+		    $Nnoexp ++;
 		    if ($debug == 1) {
 			print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n";
@@ -1311,4 +1352,35 @@
         }
     }
+    if ($debug == 1) {
+	print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n";
+    }
+
+    if ($metadata_out{nsObservingState} eq 'END_OF_NIGHT') {
+	if ($is_processing == 1) {
+	    $metadata_out{nsDiffState} = 'DIFFING';
+	}
+	elsif ($is_registering == 0) {
+	    if ($Npotential == $Nnoexp) {
+		$metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+	    }
+	    elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) {
+		$metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+	    }
+	}
+	else {
+	    $metadata_out{nsDiffState} = 'DIFFING';
+	}
+    }
+    else {
+	$metadata_out{nsDiffState} = 'DIFFING';
+    }
+
+#     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
+# 	$metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#     }	
+
+#     if ($is_processing == 1) {
+# 	$metadata_out{nsDiffState} = 'DIFFING';
+#     }
 }
 
@@ -1326,29 +1398,81 @@
 
     my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
     
     foreach my $object_row (@{ $object_ref }) {
 	my $this_object = shift @{ $object_row };
-	my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
-	$input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
-	$input_sth .= " ORDER BY dateobs ";
+#	my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#	$input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#	$input_sth .= " ORDER BY dateobs ";
 	
+	my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+	$input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+	$input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+	$input_sth .=   " ORDER BY dateobs ";
+
 	my $warps = $db->selectall_arrayref( $input_sth );
+
+	# Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+	my %comment_hash = ();
+	foreach my $this_warp (@{ $warps }) {
+	    my $this_comment = ${ $this_warp }[3];
+	    my $this_exp_id  = ${ $this_warp }[0];
+	    $comment_hash{$this_comment} = $this_exp_id;
+	}
 	
 	if (($#{ $warps } + 1) % 2 != 0) {
-	    print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
-	    next;
+	    print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+	    if ($#{ $warps} + 1 == 1) {
+		print STDERR ": I can do no diffs with only one exposure.\n";
+		next;
+	    }
+	    else {
+		print STDERR ": I should declare an exposure to be faulty.\n";
+		my @keep_warps = ();
+#		print "@{ $warps }\n";
+		foreach my $this_warp (@{ $warps }) {
+		    my $this_comment = ${ $this_warp }[3];
+		    my $this_exp_id  = ${ $this_warp }[0];
+		    if ($comment_hash{$this_comment} == $this_exp_id) {
+			push @keep_warps, $this_warp;
+		    }
+		    else {
+			print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
+		    }
+		}
+		@{ $warps } = @keep_warps;
+#		print "@{ $warps }\n";
+	    }
 	}
 	
 	while ($#{ $warps } > -1) {
 	    my $input_warp = shift @{ $warps };
+	    my $input_exp_id = ${ $input_warp }[0];
+	    my $input_comment = ${ $input_warp }[3];
+
+	    
 	    my $template_warp = shift @{ $warps };
-	    my $input_exp_id = ${ $input_warp }[0];
+
 	    my $template_exp_id = ${ $template_warp }[0];
 	    
 	    my $input_warp_id = ${ $input_warp }[1];
 	    my $template_warp_id = ${ $template_warp }[1];
+
+	    my $input_warp_state = ${ $input_warp }[4];
+	    my $template_warp_state = ${ $template_warp }[4];
 	    
+	    $Npotential++;
+
+	    unless (defined($input_warp_id) && defined($template_warp_id) &&
+		    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+		print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed\n";
+		next;
+	    }
+
 	    if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
-		print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
+		$Nqueued++;
+		print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
 		next;
 	    }
@@ -1366,4 +1490,5 @@
 		$cmd .= ' -pretend ';
 		print STDERR "diff_queue: $cmd\n";
+		print STDERR " $input_warp_id $template_warp_id\n";
 	    }
 	    
@@ -1375,7 +1500,14 @@
 		    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
 		}
-	    }
-	}
-    }
+		$Nqueued++;
+	    }
+	}
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#  	$metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }	
+
 }
 	    
@@ -1493,5 +1625,5 @@
 	}
     }	
-
+#    print "$datetime $eon_dt\n";
     if (DateTime->compare($datetime,$eon_dt) < 1) {
 	return("OBSERVING");
@@ -1501,4 +1633,33 @@
     }
 }
+
+# This basically does the end of night check, but does it "the hard way," to prevent the time from fooling us.
+sub get_registration_state {
+    my $date = shift;
+
+    foreach my $eon (keys %eon_config) {
+	my $command = "$regtool -processedexp -simple ";
+	$command .= " -dbname $dbname ";
+	$command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
+	$command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT});
+	$command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE});
+	$command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE});
+	$command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT});
+
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run ( command => $command, verbose => $verbose );
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+	}
+	my @eon_exposures = split /\n/, (join '', @$stdout_buf);
+	if ($#eon_exposures >= 0) {
+	    return("REGISTERED");
+	}
+    }	
+    return("NOT_REGISTERED");
+}
+
+    
 
 sub get_tool_parameters {
@@ -1620,6 +1781,6 @@
     my $stage = shift; # stage name
     my $stage_id = shift; #  identifier
+    my $date = shift;
     my $exit_code = shift; # Exit code
-    # outputImage and path_base are globals
 
     carp($msg);
Index: branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl	(revision 30586)
@@ -42,5 +42,5 @@
 
 # Parse the command-line arguments
-my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
+my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir, $need_magic );
 my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
 my ( $output_format );
@@ -54,4 +54,5 @@
     'fileset=s'         => \$fileset,     # Fileset name
     'workdir=s'         => \$workdir,     # Working directory
+    'need-magic'        => \$need_magic,  # do we require censored detections?
     'dbname=s'          => \$dbname,    # Database name
     'verbose'           => \$verbose,   # Print to stdout
@@ -166,4 +167,10 @@
     foreach my $comp ( @$components ) {
         my $path_base = $comp->{path_base}; # Base name for file
+        if (!$need_magic and $comp->{magicked}) {
+            # This client is authorized to receive uncensored detections
+            # Get the uri for the "backup" files
+            print "Using uncensored input from $path_base\n";
+            $path_base = $ipprc->destreaked_filename($path_base);
+        }
         next if defined $comp->{quality} and $comp->{quality} > 0;
         print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
@@ -181,5 +188,10 @@
             $filename = $ipprc->filename( "PSPHOT.OUT.CMF.MEF", "$path_base.pos" );
         }
-        $filename = $ipprc->file_resolve($filename);
+        &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename);
+        my $resolved = $ipprc->file_resolve($filename);
+        
+        &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved;
+
+        $filename = $resolved;
 
         my $data = { zp => $zp,
Index: branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl	(revision 30586)
@@ -348,4 +348,10 @@
     if (defined $exp_id and not $no_update) {
         my $command = "$regtool -addprocessedexp -exp_id $exp_id -fault $exit_code";
+	$command .= " -exp_tag $exp_tag";
+	$command .= " -exp_name UNKNOWN";
+	$command .= " -inst UNKNOWN";
+	$command .= " -telescope UNKNOWN";
+	$command .= " -telescope UNKNOWN";
+	$command .= " -filelevel UNKNOWN";
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;
Index: branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl	(revision 30419)
+++ branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl	(revision 30586)
@@ -10,5 +10,6 @@
 my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host at $date\n\n";
+my $cmd_line = join ' ', @ARGV;
+print "Starting script $0 $cmd_line on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -39,4 +40,5 @@
 
 my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile);
+my ($sunrise, $sunset);
 GetOptions(
     'caches'           => \$cache,
@@ -47,4 +49,6 @@
     'bytes=s'          => \$bytes,
     'md5sum=s'         => \$md5sum,
+    'sunrise=s'        => \$sunrise,
+    'sunset=s'         => \$sunset,
     'dbname|d=s'       => \$dbname,    # Database name
     'verbose'          => \$verbose,   # Print to stdout
@@ -54,6 +58,6 @@
 ) or pod2usage( 2 );
 
-my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
+my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -65,4 +69,10 @@
     defined $uri;
 
+unless (defined($sunset)) {
+    $sunset = '03:30:00';
+}
+unless (defined($sunrise)) {
+    $sunrise = '17:30:00';
+}
 
 my $RECIPE = "REGISTER"; # Recipe to use for ppStats
@@ -84,5 +94,6 @@
 
 my $cmdflags;
-my $burntoolStateGood;
+my $burntoolStateTarget;
+my $burntoolStateCurrent;
 # Run ppStats on the input file
 {
@@ -101,5 +112,5 @@
     print "STDOUT:\n$out1";
     print "STDERR:\n$err1";
-    &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
+    &my_die_for_add("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
 
     print "[Running " . join(' ', @command2) . "]\n";
@@ -110,16 +121,16 @@
     print "STDOUT:\n$out2";
     print "STDERR:\n$err2";
-    &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
+    &my_die_for_add("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
     chomp $out2;
     $cmdflags = $out2;
 
     # Manually parse the burntool_state entry.
-    my $burntoolState = 0;
+    $burntoolStateCurrent = 0;
     my $isGPC1 = 0;
-    $burntoolStateGood = 0;
+    $burntoolStateTarget = 0;
     foreach my $line (split /\n/, $out1) {
         if ($line =~ /FPA.BURNTOOL.APPLIED/) {
             $line =~ s/^\s+//;
-            $burntoolState = (split /\s+/, $line)[2];
+            $burntoolStateCurrent = (split /\s+/, $line)[2];
         }
         if ($line =~ /FPA.CAMERA/) {
@@ -131,8 +142,8 @@
     }
     if ($isGPC1 != 1) {
-        $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool.
-    }
-    elsif (($isGPC1 == 1) && ($burntoolState == 1)) {
-#       print STDERR "In the good region: >>$burntoolState<<\n";
+        $burntoolStateCurrent = 0; # If it's not GPC1, you shouldn't have run burntool.
+    }
+    elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) {
+#       print STDERR "In the good region: >>$burntoolStateCurrent<<\n";
         my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -148,23 +159,26 @@
             if ($line =~ /BURNTOOL.STATE.GOOD/) {
                 $line =~ s/^\s+//;
-                $burntoolStateGood = (split /\s+/, $line)[2];
+                $burntoolStateTarget = (split /\s+/, $line)[2];
                 last;
             }
         }
-        $burntoolState = $burntoolStateGood; # Positive because this has the header table.
-
-    }
-    $cmdflags .= " -burntool_state $burntoolState ";
+        # XXX why was this being equated??
+        # $burntoolState = $burntoolStateGood; # Positive because this has the header table.
+
+    }
+    $cmdflags .= " -burntool_state $burntoolStateTarget ";
 }
 
 $now_time = localtime();
 printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose;
+printf STDERR "\nburntool state current: %d target: %d\n", $burntoolStateCurrent, $burntoolStateTarget;
 
 # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
-if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die_for_add ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die_for_add ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die_for_add ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die_for_add ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die_for_add ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
 
 my $command = "$regtool -addprocessedimfile";
@@ -177,5 +191,16 @@
 $command .= " -hostname $host" if defined $host;
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= " -data_state check_burntool";
+if (abs($burntoolStateCurrent) == $burntoolStateTarget) {
+    printf STDERR "This has already been burned.\n";
+    $command .= " -data_state full";
+}
+elsif (is_daytime($dateobs,$sunrise,$sunset)) {
+    printf STDERR "This is a daytime exposure.\n";
+    $command .= " -data_state full";
+}
+else {
+    printf STDERR "Need to check burntool.\n";
+    $command .= " -data_state check_burntool";
+}
 $command .= " $cmdflags";
 
@@ -187,5 +212,5 @@
 my $ra        = &value_for_flag($cmdflags, 0.0, "-ra");
 my $dec       = &value_for_flag($cmdflags, 0.0, "-decl");
-my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
+
 
 # if the needed data is available, pass it to sunmoon:
@@ -216,5 +241,5 @@
 # lock file?
 # if exp_type = DARK and date > MIDNIGHT HST { wait }
-# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 
+# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log");
 
 $now_time = localtime();
@@ -236,5 +261,6 @@
 # We now have an imfile in the database, check if we can burntool it.  If not, continue on.
 
-{ 
+if ((abs($burntoolStateCurrent) != $burntoolStateTarget)&&
+    (!is_daytime($dateobs,$sunrise,$sunset))) {
     my $mdcParser  = PS::IPP::Metadata::Config->new;
 
@@ -243,19 +269,20 @@
     $check_date =~ s/T.*$//;
     my $exp_name   = $tmp_exp_name;
-    
+
 
     my $bt_check_command = "$regtool -checkburntoolimfile ";
     $bt_check_command .= " -class_id $class_id ";
-    $bt_check_command .= " -date $check_date ";
-    $bt_check_command .= " -valid_burntool $burntoolStateGood ";
+    $bt_check_command .= " -dateobs_begin ${check_date}T${sunset} ";
+    $bt_check_command .= " -dateobs_end ${check_date}T${sunrise} ";
+    $bt_check_command .= " -valid_burntool $burntoolStateTarget ";
     $bt_check_command .= " -exp_name $exp_name ";
     $bt_check_command .= " -dbname $dbname" if defined $dbname;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run ( command => $bt_check_command, verbose => $verbose);
+        run ( command => $bt_check_command, verbose => $verbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform regtool: $error_code", 
-		$class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR);
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die_for_update("Unable to perform regtool: $error_code",
+                           $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
 
@@ -264,20 +291,20 @@
     my $exposures;
     while ( scalar @whole > 0 ) {
-	my $value = shift @whole;
-	push @single, $value;
-	if ($value =~ /^\s*END\s*$/) {
-	    push @single, "\n";
-	    
-	    my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
-	    &my_die("Unable to parse output from regtool", 
-		    $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless
-			defined $list;
-	    push @{ $exposures }, @$list;
-	    @single = ();
-	}
-    }
-    
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
+            &my_die_for_update("Unable to parse output from regtool",
+                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless
+                                   defined $list;
+            push @{ $exposures }, @$list;
+            @single = ();
+        }
+    }
+
     # We only care about the final entry, as that contains the exposure we are.
-    
+
     my $regtool_update = "$regtool -updateprocessedimfile ";
     $regtool_update .= "-dbname $dbname " if defined $dbname;
@@ -286,35 +313,35 @@
     my $burntool_data = pop(@{ $exposures });
     if ($burntool_data->{burnable} == 0) {
-	$regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
-	unless ($no_update) {
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		warn ("Unable to perform regtool -addprocessedimfile: $error_code");
-		exit($error_code);
-	    }
-	} else {
-	    print "skipping command: $command\n";
-	}
+        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+                exit($error_code);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
     }
     else {
-	my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
-	$apply_command .= " --class_id $class_id --exp_id $exp_id ";
-	$apply_command .= " --this_uri $burntool_data->{uri} ";
-	$apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
-	$apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
-	$apply_command .= " --verbose " if $verbose;
-	print "$apply_command\n";
-	unless ($no_update) {
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		IPC::Cmd::run(command => $apply_command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
-		exit($error_code);
-	    }
-	}
-    }    
+        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
+        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
+        $apply_command .= " --this_uri $burntool_data->{uri} ";
+        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
+        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
+        $apply_command .= " --verbose " if $verbose;
+        print "$apply_command\n";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die_for_update("Unable to perform ipp_apply_burntool_single.pl: $error_code",
+                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+        }
+    }
 
 
@@ -353,5 +380,74 @@
 }
 
-sub my_die
+sub is_daytime
+{
+    my $dateobs = shift;
+    my $sunset  = shift;
+    my $sunrise = shift;
+
+    my ($date,$time);
+
+    if ($dateobs =~ /T/) {
+	($date,$time) = split /T/, $dateobs;
+    }
+    else {
+	($date,$time) = split / /, $dateobs;
+    }
+    my ($hour,$minute,$second) = split /\:/, $time; # /;
+    my ($ss_hour,$ss_minute,$ss_second) = split /\:/, $sunset; # /;
+    my ($sr_hour,$sr_minute,$sr_second) = split /\:/, $sunrise; # /;
+    if ($second =~ /Z/) {
+	$second =~ s/Z//;
+    }
+#     print "this exposure: $hour $minute $second\n";
+#     print "sunset:        $ss_hour $ss_minute $ss_second\n";
+#     print "sunrise:       $sr_hour $sr_minute $sr_second\n";
+#     printf "Hss: %d Mss: %d Sss: %d\n",($hour >= $ss_hour),($minute >= $ss_minute),($second >= $ss_second);
+#     printf "Hsr: %d Msr: %d Ssr: %d\n",($hour <= $sr_hour),($minute <= $sr_minute),($second <= $sr_second);
+
+    if (($hour > $ss_hour)&&($hour <= 24)) {
+	return(1); # After sunset by more than an hour, before midnight
+    }
+    elsif ($hour == $ss_hour) {
+	if ($minute > $ss_minute) {
+	    return(1); # After sunset by more than a minute
+	}
+	elsif ($minute == $ss_minute) {
+	    if ($second >= $ss_second) {
+		return(1); # After sunset by more than a second
+	    }
+	    else {
+		return(0);
+	    }
+	}
+	else {
+	    return(0);
+	}
+    }
+    elsif (($hour < $sr_hour)&&($hour >= 0)) {
+	return(1); # Before sunrise by more than an hour, but after midnight
+    }
+    elsif ($hour == $sr_hour) {
+	if ($minute < $sr_minute) {
+	    return(1); # Before sunrise by more than a minute
+	}
+	elsif ($minute == $sr_minute) {
+	    if ($second <= $sr_second) {
+		return(1); # Before sunrise by more than a second
+	    }
+	    else {
+		return(0);
+	    }
+	}
+	else {
+	    return(0);
+	}
+    }
+    else {
+	return(0); # We should never get here.
+    }
+}
+
+sub my_die_for_add
 {
     my $msg = shift; # Warning message on die
@@ -384,4 +480,26 @@
     exit $exit_code;
 }
+sub my_die_for_update
+{
+    my $msg = shift; # Warning message on die
+    my $exp_id = shift;
+    my $class_id = shift;
+    my $exit_code = shift;
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
+        my $command = "$regtool -updateprocessedimfile";
+        $command .= " -exp_id $exp_id";
+        $command .= " -class_id $class_id";
+        $command .= " -fault $exit_code";
+        $command .= " -hostname $host" if defined $host;
+        $command .= " -dbname $dbname" if defined $dbname;
+        print "Running: $command\n";
+        system($command);
+    }
+    exit $exit_code;
+}
 
 # Pau.