Index: branches/czw_branch/20101203/ippScripts/scripts/automate_stacks.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/automate_stacks.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/automate_stacks.pl	(revision 30118)
@@ -1355,5 +1355,5 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
 
-    if ($target eq 'OSS') {
+    if (($target eq 'OSS')||($target eq 'SweetSpot')) {
 	my $db = init_gpc_db();
 
@@ -1372,6 +1372,7 @@
 	    
 	    if (($#{ $warps } + 1) % 2 != 0) {
-		print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
-		last;
+		print STDERR "Number of input warps to make OSS diffs is not even! $this_object $#{ $warps }\n";
+#		last;
+		next;
 	    }
 	    
Index: branches/czw_branch/20101203/ippScripts/scripts/camera_exp.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/camera_exp.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/camera_exp.pl	(revision 30118)
@@ -378,5 +378,6 @@
         $ipprc->file_exists($file);
 
-    if ($replicate and (file_scheme($file) eq 'neb')) {
+    my $scheme = file_scheme($file);
+    if ($replicate and $scheme and (file_scheme($file) eq 'neb')) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $cam_id, $PS_EXIT_SYS_ERROR);
     }
Index: branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl	(revision 30118)
@@ -159,5 +159,4 @@
         if (storage_object_exists($configuration, \$gone)) {
             if ($gone) {
-                rename_gone_file($configuration);
                 $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
                 # if we dump the config we need to insure that the config dump represents
@@ -177,4 +176,7 @@
     $ipprc->delete_destreak_backup_file($outputWeight)
         or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # don't do binned images when updating unless we are starting from scratch.
+    $do_binned_images = 0 unless $dump_config;
 }
 if ($do_binned_images) {
@@ -491,5 +493,7 @@
         check_output($configuration, 1) if $dump_config;
         check_output($backmdl, 1) if $outputBackmdlExpect;
-        check_output($pattern, 1) if $outputPatternExpect;
+	# allow the pattern file to be missing if run state is update older data doesn't have one
+	# I should parse the config dump file to calculate the 'Expect' variables
+        check_output($pattern, 1, $run_state eq 'update') if $outputPatternExpect;
         if ($do_photom) {
             check_output($outputSources, 1);
@@ -556,5 +560,4 @@
             if ($gone) {
                 carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
-                rename_gone_file($outputSources);
                 $make_sources = 1;
             }
@@ -579,5 +582,4 @@
             if ($gone) {
                 carp "WARNING: PSF storage object exists but all instances are permanently gone";
-                rename_gone_file($outputPsf);
                 $make_psf = 1;
             }
@@ -613,6 +615,9 @@
 sub storage_object_exists
 {
+    return 0 if !$neb;
+
     my $file = shift;
     my $ref_all_gone = shift;
+
 
     my $exists = $neb->storage_object_exists($file);
@@ -678,14 +683,4 @@
     return 1;
 }
-sub rename_gone_file
-{
-    # check whether the only instance of file is on a lost volume
-    # XXX: we don't have a proper interface for this. 
-    # For now try to use Bill's hack: the script 'whichnode'
-
-    my $file = shift;
-    $neb->move($file, "$file.gone") or 
-                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-}
 
 # Prepare to write to an output file
@@ -711,4 +706,5 @@
     my $file = shift;
     my $replicate = shift;
+    my $allow_missing = shift;
 
     if (!defined $file) {
@@ -716,7 +712,15 @@
     }
 
-    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
-
-    if ($replicate and (file_scheme($file) eq 'neb')) {
+    my $exists = $ipprc->file_exists($file);
+
+    if (!$exists) {
+        if ($allow_missing) {
+            carp("Couldn't find expected output_file: $file but continuing anyways\n");
+            return 1;
+        }
+        &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+
+    if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
Index: branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/dist_bundle.pl	(revision 30118)
@@ -62,4 +62,6 @@
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $file_cmd   = can_run('file') or (warn "can't find program file" and $missing_tools = 1);
+my $zcat   = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -526,8 +528,42 @@
                     $PS_EXIT_CONFIG_ERROR) if (!$resolved);
 
+    &my_die("config dump file resolved but not accessible: $config_file_rule", $component,
+                    $PS_EXIT_CONFIG_ERROR) if !$ipprc->file_exists($resolved);
+
+    my $mdc_compressed;
+    {
+        my $command = "$file_cmd $resolved";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        my $output = join "", @$stdout_buf;
+        # XXX: may need to to make this more robust
+        $mdc_compressed = ($output =~ /gzip/);
+    }
+    my $inName;
+    if ($mdc_compressed) {
+        my $tmpfile;
+        ($tmpfile, $inName) = tempfile( "/tmp/bundle.XXXX", UNLINK => !$save_temps );
+        close($tmpfile) or &my_die("failed to close $inName", $component, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $command = "$zcat $resolved > $inName";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+
+
+    } else {
+        $inName = $resolved;
+    }
+    my $in = open_with_retries($inName);
+
     # we don't use the mdc parser because the perl parser is way is too slow for complicated config
     # files like this
-    my $in = open_with_retries($resolved);
-
     my $line;
     while ($line = <$in>) {
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak.pl	(revision 30118)
@@ -309,5 +309,8 @@
         }
 
-        $sources = $ipprc->filename("PSPHOT.OUTPUT",  $path_base, $class_id);
+        # only destreak cmf file if this is a new run
+        if ($run_state eq 'new') {
+            $sources = $ipprc->filename("PSPHOT.OUTPUT",  $path_base, $class_id);
+        }
 
     } elsif ($stage eq "chip_bg") {
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_defineruns.pl	(revision 30118)
@@ -78,5 +78,5 @@
     foreach my $label (@labels) {
         my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";
Index: branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_revert.pl	(revision 30118)
@@ -191,5 +191,4 @@
     $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
     $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-    $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id);
 
     # we use the mask output from the camera stage for input and replace
@@ -204,4 +203,10 @@
     }
 
+    # only revert sources if this is a new run, not one being updated
+    if ($run_state eq 'new') {
+        $sources = $ipprc->filename("PSPHOT.OUTPUT", $path_base, $class_id);
+        $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id);
+    }
+
     $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id);
     # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
@@ -209,5 +214,4 @@
     $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
-    $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $class_id);
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
Index: branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/publish_file.pl	(revision 30118)
@@ -44,4 +44,5 @@
 my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
 my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
+my ( $output_format );
 
 GetOptions(
@@ -59,4 +60,5 @@
     'save-temps'        => \$save_temps, # Save temporary files?
     'redirect-output'   => \$redirect,   # Redirect output to log file?
+    'output_format=i'   => \$output_format, # Output format for ppMops
     ) or pod2usage( 2 );
 
@@ -191,4 +193,5 @@
                      warp_id => $comp->{warp1},
                      diff_id => $comp->{diff_id},
+                     output_format => $comp->{output_format},
                      direction => 1,
         };
@@ -323,5 +326,4 @@
 
     my $command = "$ppMops $input $output";
-    $command .= " -version 1"; # XXX : NOTE: for now, MOPS just wants PS1_DV1 (remove this eventually...)
     $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
     $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
@@ -335,4 +337,5 @@
     $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
     $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
+    $command .= " -version " . $data->{output_format} if defined $data->{output_format};
 
     unless ($no_op) {
Index: branches/czw_branch/20101203/ippScripts/scripts/receive_file.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/receive_file.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/receive_file.pl	(revision 30118)
@@ -32,4 +32,5 @@
 my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
 my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1);
+my $gunzip = can_run('gunzip') or (warn "Can't find gunzip" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -404,4 +405,24 @@
     my $workdir = shift;
 
+    my $filecmd_output = `file $src`;
+    &my_die("failed to determinte file type of $src", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$filecmd_output;
+    chomp $filecmd_output;
+
+    my $tmpName;
+    if ($filecmd_output =~ /gzip/) {
+        my $tmpfile;
+        ($tmpfile, $tmpName) = tempfile( "/tmp/receive.XXXX", UNLINK => !$save_temps );
+        close($tmpfile) or &my_die("failed to close $tmpName", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+        my $command = "$gunzip -c $src > $tmpName";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $component, $error_code);
+        }
+        $src = $tmpName;
+    }
+
     open my $IN,  "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR);
     open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR);
Index: branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl
===================================================================
--- branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl	(revision 30047)
+++ branches/czw_branch/20101203/ippScripts/scripts/register_imfile.pl	(revision 30118)
@@ -287,4 +287,15 @@
     if ($burntool_data->{burnable} == 0) {
 	$regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+	unless ($no_update) {
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+		exit($error_code);
+	    }
+	} else {
+	    print "skipping command: $command\n";
+	}
     }
     else {
@@ -296,24 +307,15 @@
 	$apply_command .= " --verbose " if $verbose;
 	print "$apply_command\n";
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    IPC::Cmd::run(command => $apply_command, verbose => $verbose);
-	unless ($success) {
-	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	    warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
-	    exit($error_code);
+	unless ($no_update) {
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
+		exit($error_code);
+	    }
 	}
     }    
 
-    unless ($no_update) {
-	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	    IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
-	unless ($success) {
-	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	    warn ("Unable to perform regtool -addprocessedimfile: $error_code");
-	    exit($error_code);
-	}
-    } else {
-	print "skipping command: $command\n";
-    }
 
 }