Index: /branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl	(revision 24784)
+++ /branches/czw_branch/cleanup/ippScripts/scripts/ipp_cleanup.pl	(revision 24785)
@@ -747,5 +747,709 @@
 
 } 
-# fake : faketool : -pendingcleanupimfile (loop over imfiles)
+# Detrend stages
+if ($stage eq "detrend.process.imfile") {
+
+    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find chiptool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_processedimfile -chip_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
+    }
+
+    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+	my $command = "$dettool -det_id $stage_id -updateprocessedimfile -data_state new";
+	$command .= " -dbname $dbname" if defined $dbname;
+
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	}
+	exit 0;
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $imfiles = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+	my $exp_id   = $imfile->{exp_id};
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+	# goto_scrubbed now requires the config file to not exist.
+	
+	# Possibly not the correct config file, but simtest doesn't leave any around to check.
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
+                    . " because config file is missing\n";
+                $status = 0;
+            }
+        }
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
+		    . " because config file is present\n";
+		$status = 0;
+	    }
+	}
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+	    elsif ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+	    elsif ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+
+	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+	    my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+if ($stage eq "detrend.process.exp") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $camtool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+
+    foreach my $exp (@$exps) {
+	my $path_base = $exp->{path_base};
+	my $exp_id    = $exp->{exp_id};
+
+	my $status = 1;
+	# don't clean up unless the data needed to update is available
+	# goto_scrubbed now requires the config file to not be present
+	if ($mode eq "goto_cleaned") {
+	    my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
+	    
+	    if (!$config_file or ! -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+		$status = 0;
+	    }
+	}
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
+	    
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+		$status = 0;
+	    }
+	}
+	if ($status) {
+	    my @files = ();
+	    # delete the temporary image datafiles
+	    addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
+	    if ($mode eq "goto_purged") {
+		# additional files to remove for 'purge' mode
+		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
+		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
+		addFilename (\@files, "PSASTRO.STATS", $path_base);
+	    }
+	    # actual command to delete the files
+	    $status = &delete_files (\@files);
+	}
+	
+	if ($status)  {
+	    my $command;
+	    if ($mode eq "goto_cleaned") {
+		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
+	    }
+	    if ($mode eq "goto_scrubbed") {
+		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
+	    }
+	    if ($mode eq "goto_purged") {
+		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	} else {
+	    # since 'camera' has only a single imfile, we can just update the run
+	    my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	    exit $PS_EXIT_UNKNOWN_ERROR;
+	}
+    }
+    exit 0;
+}
+if ($stage eq "detrend.stack.imfile") {
+    
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+
+    # this stage uses 'stacktool'
+    my $stacktool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    my $skyfiles;                  # Array reference of component files
+    my $command = "dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 
+	&my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $skyfiles = parse_md_list($metadata) or 
+	&my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+    
+    my @files = ();
+    foreach my $stack (@{ $stacks }) {
+	# detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
+	my $path_base = $stack->{uri};
+	my $iteration = $stack->{iteration};
+	my $class_id  = $stack->{class_id};
+
+	$path_base =~ s/\.fits$//; # That should do it?
+
+	my $status = 1;
+	if ($mode eq "goto_cleaned") {
+	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
+
+	    $config_file =~ s%^file://%%;
+	    if (!$config_file or ! -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id $path_base" .
+		    " because config file is missing\n";
+		$status = 0;
+	    }
+	    $config_file = 'file://' . $config_file;
+	}
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
+	    $config_file =~ s%^file://%%;
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
+		    " because config file is present\n";
+		$status = 0;
+	    }
+	}
+	if ($status) {
+	    # delete the temporary image datafiles
+	    addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $skycell_id);
+	    addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $skycell_id);
+	    addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $skycell_id);
+
+	    if ($mode eq "goto_purged") {
+		# additional files to remove for 'purge' mode
+		# Nothing?
+	    }
+
+	    $status = &delete_files(\@files);
+	}
+
+	if ($status) {
+	    my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
+	    if ($mode eq "goto_purged") {
+		$command .= " -updatestacked -data_state purged";
+	    } 
+	    elsif ($mode eq "goto_cleaned") {
+		$command .= " -updatestacked -data_state cleaned";
+	    }
+	    elsif ($mode eq "goto_scrubbed") {
+		$command .= " -updatestacked -data_state scrubbed";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	} else {
+	    my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	    exit $PS_EXIT_UNKNOWN_ERROR;
+	}
+    }
+    exit 0;
+}
+if ($stage eq "detrend.normstat.imfile") {
+    print STDERR "I'm not convinced there's anything to clean up from stage $stage\n";
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $camtool = can_run('dettool') or die "Can't find dettool";
+
+    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+	my $path_base = $exp->{path_base};
+	my $iteration = $exp->{iteration};
+	my $class_id  = $exp->{class_id};
+
+	my $status = 1;
+	if ($status)  {
+	    my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
+	    if ($mode eq "goto_cleaned") {
+		$command .= " -data_state cleaned";
+	    }
+	    if ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+	    if ($mode eq "goto_purged") {
+		$command .= " -data_state purged";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	} else {
+	    # since 'camera' has only a single imfile, we can just update the run
+	    my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	    exit $PS_EXIT_UNKNOWN_ERROR;
+	}
+    }
+    exit 0;
+
+}
+if ($stage eq "detrend.norm.imfile") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $camtool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+	my $path_base = $exp->{path_base};
+	my $iteration = $exp->{iteration};
+	my $class_id  = $exp->{class_id};
+
+	my $status = 1;
+	# don't clean up unless the data needed to update is available
+	# goto_scrubbed now requires the config file to not be present
+	if ($mode eq "goto_cleaned") {
+	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+	    
+	    if (!$config_file or ! -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+		$status = 0;
+	    }
+	}
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+	    
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+		$status = 0;
+	    }
+	}
+	if ($status) {
+	    my @files = ();
+	    # delete the temporary image datafiles
+	    addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
+	    addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
+	    addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
+	    if ($mode eq "goto_purged") {
+		# additional files to remove for 'purge' mode
+		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
+		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
+	    }
+	    # actual command to delete the files
+	    $status = &delete_files (\@files);
+	}
+	
+	if ($status)  {
+	    my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
+	    if ($mode eq "goto_cleaned") {
+		$command .= " -data_state cleaned";
+	    }
+	    if ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+	    if ($mode eq "goto_purged") {
+		$command .= " -data_state purged";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	} else {
+	    my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	    exit $PS_EXIT_UNKNOWN_ERROR;
+	}
+    }
+    exit 0;
+}
+if ($stage eq "detrend.resid.imfile") {
+
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_residimfile -chip_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
+    }
+
+    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+	my $command = "$dettool -det_id $stage_id -updateresidimfile -data_state new";
+	$command .= " -dbname $dbname" if defined $dbname;
+
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	}
+	exit 0;
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $imfiles = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+        my $class_id = $imfile->{class_id};
+	my $iteration = $imfile->{iteration};
+	my $exp_id = $imfile->{exp_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+	# goto_scrubbed now requires the config file to not exist.
+	
+	# Possibly not the correct config file, but simtest doesn't leave any around to check.
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id $class_id "
+                    . " because config file is missing\n";
+                $status = 0;
+            }
+        }
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping scrubbed for $stage $stage_id $class_id "
+		    . " because config file is present\n";
+		$status = 0;
+	    }
+	}
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+	    elsif ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+	    elsif ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+	    my $command = "$dettool -updateresidimfile -chip_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+if ($stage eq "detrend.resid.exp") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $camtool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+	my $exp_id = $exp->{exp_id};
+	my $iteration = $exp->{iteration};
+	my $path_base = $exp->{path_base};
+
+	my $status = 1;
+	# don't clean up unless the data needed to update is available
+	# goto_scrubbed now requires the config file to not be present
+	if ($mode eq "goto_cleaned") {
+	    my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
+	    
+	    if (!$config_file or ! -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+		$status = 0;
+	    }
+	}
+	elsif ($mode eq "goto_scrubbed") {
+	    my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
+	    
+	    if ($config_file and -e $config_file) {
+		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+		$status = 0;
+	    }
+	}
+	if ($status) {
+	    my @files = ();
+	    # delete the temporary image datafiles
+	    addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
+	    if ($mode eq "goto_purged") {
+		# additional files to remove for 'purge' mode
+		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
+		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
+		addFilename (\@files, "PSASTRO.STATS", $path_base);
+	    }
+	    # actual command to delete the files
+	    $status = &delete_files (\@files);
+	}
+	
+	if ($status)  {
+	    my $command = "$dettool -updateresiddexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
+	    if ($mode eq "goto_cleaned") {
+		$command .= " -data_state cleaned";
+	    }
+	    if ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+	    if ($mode eq "goto_purged") {
+		$command .= " -data_state purged";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	} else {
+	    my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
+	    $command .= " -dbname $dbname" if defined $dbname;
+	    
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+	    }
+	    exit $PS_EXIT_UNKNOWN_ERROR;
+	}
+    }
+    exit 0;
+}
+
 
 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
Index: /branches/czw_branch/cleanup/ippTools/src/dettool.c
===================================================================
--- /branches/czw_branch/cleanup/ippTools/src/dettool.c	(revision 24784)
+++ /branches/czw_branch/cleanup/ippTools/src/dettool.c	(revision 24785)
@@ -1817,4 +1817,15 @@
     if (!strcmp(data_state, "drop")) return true;
     if (!strcmp(data_state, "register")) return true;
+    // These are valid data states, and are necessary for the cleanup to work correctly.
+    if (!strcmp(data_state, "full")) return true;
+    if (!strcmp(data_state, "goto_cleaned")) return true;
+    if (!strcmp(data_state, "goto_scrubbed")) return true;
+    if (!strcmp(data_state, "goto_purged")) return true;
+    if (!strcmp(data_state, "cleaned")) return true;
+    if (!strcmp(data_state, "scrubbed")) return true;
+    if (!strcmp(data_state, "purged")) return true;
+    if (!strcmp(data_state, "error_cleaned")) return true;
+    if (!strcmp(data_state, "error_scrubbed")) return true;
+    if (!strcmp(data_state, "error_purged")) return true;
 
     psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state);
Index: /branches/czw_branch/cleanup/ippTools/src/dettoolConfig.c
===================================================================
--- /branches/czw_branch/cleanup/ippTools/src/dettoolConfig.c	(revision 24784)
+++ /branches/czw_branch/cleanup/ippTools/src/dettoolConfig.c	(revision 24785)
@@ -41,11 +41,11 @@
     // XXX EAM : is this used?  does it make sense?
     psMetadata *pendingArgs = psMetadataAlloc();
-    psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exposure type", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,            "search by camera", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,            "search by filter", NULL);
+    psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,         "search by exposure ID", 0);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,       "search by exposure type", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,           "search by camera", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,      "search by telescope", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,         "search by filter", NULL);
     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri",  0,            "search by URL", NULL);
-    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,        "use the simple output format", false);
 
     // -definebytag
@@ -195,7 +195,7 @@
     // -addcorrectimfile
     psMetadata *addcorrectimfileArgs = psMetadataAlloc();
-    psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "define detrend ID (required)", 0);
-    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search for class ID (required)", NULL);
-    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,            "define resid file URI", NULL);
+    psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,        "define detrend ID (required)", 0);
+    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,      "search for class ID (required)", NULL);
+    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,           "define resid file URI", NULL);
     psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
     psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev",  0,            "define exposure background stdev", NAN);
@@ -294,5 +294,5 @@
     psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
     psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_processedimfile
@@ -354,5 +354,5 @@
     psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
     psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
-    psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_processedexp
@@ -414,5 +414,5 @@
     psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_stacked
@@ -466,5 +466,5 @@
     psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedstat
@@ -528,5 +528,5 @@
     psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedimfile
@@ -586,5 +586,5 @@
     psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
     psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
-    psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedexp
@@ -666,5 +666,5 @@
     psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
     psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_residimfile
@@ -747,5 +747,5 @@
     psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
     psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject",  0,            "exposure is not to be stacked in the next iteration", false);
-    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_residexp