Index: trunk/tools/stsci_disks/p0_large.pl
===================================================================
--- trunk/tools/stsci_disks/p0_large.pl	(revision 38680)
+++ trunk/tools/stsci_disks/p0_large.pl	(revision 38682)
@@ -76,5 +76,5 @@
 
 my $time = time();
-my $logDest = "neb://any/stsci_shuffle/large/${host}.${volume}/${dir}.log.${time}";
+my $logDest = "neb://any/stsci_shuffle/sizes/${host}.${volume}/${dir}.log";
 my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n";
 $ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n";
@@ -104,4 +104,7 @@
 my $root = "/data/${host}.${volume}/nebulous/${dir}";
 my @subdirectories = <$root/*>;
+my $size = 0;
+my $limit = 10 * 1024 * 1024; # stat returns bytes, so 100 * meg * k 
+my ($size_ota,$size_pv1,$size_pv2,$size_md) = (0,0,0,0);
 foreach my $subdir (@subdirectories) {
     my @files = <${subdir}/*>;
@@ -110,46 +113,60 @@
 	
 	my @stat = stat($f);
-	my $limit = 10 * 1024 * 1024; # stat returns bytes, so 100 * meg * k 
+	$size += $stat[7];
 #	print "$f $stat[7]\n"; die;
-	if ($stat[7] > $limit) {
-	    my $ins_id = $f;
-	    $ins_id =~ s/^$subdir\///;
-	    $ins_id =~ s/\..*//;
+	if ($f =~ /ota/) { $size_ota += $stat[7]; }
+	if ($f =~ /LAP.ThreePi.20120706/) { $size_pv1 += $stat[7]; }
+	if ($f =~ /LAP.ThreePi.20130717/) { $size_pv2 += $stat[7]; }
+	if ($f =~ /MD/) { $size_md += $stat[7]; }
 
-	    my $key = $f;
-	    $key =~  s/.*\/${ins_id}\.//;
-	    $key =~  s%:%/%g;
+
+# 	if ($stat[7] > $limit) {
+# 	    if (0) {
+# 		my $ins_id = $f;
+# 		$ins_id =~ s/^$subdir\///;
+# 		$ins_id =~ s/\..*//;
 		
-	    my $uris = $neb->find_instances($key);
-	    my $found = 0;
-	    foreach my $uri (@{ $uris }) {
-		if ($uri eq "file://${f}") {
-		    $found = 1;
-		}
-	    }
-	    if ($#{ $uris } == -1) { # true orphan
-		if ($key =~ /ecliptic/) {
-		    $key =~ s%/00hr%:00hr%;
-		    $neb->replicate($key);
-		    print "#replicate $key\n";
-		}
-		$found = -1;
-	    }
-	    print "$f $stat[7] $key $found\n";
-	    if ($found == 0) {
-#		print "rm $f\n";
-		unlink($f);
-		foreach my $rsync (@rsync_vols) {
-		    my $test_f = $f;
-		    $test_f =~ s%^/data/%%;
-		    $test_f = "/data/${rsync}/stsci_shuffle/${test_f}";
-		    if (-e $test_f) {
-			unlink($f);
-#			print "rm $test_f\n";
-		    }
-		}
-	    }
-	}
+# 		my $key = $f;
+# 		$key =~  s/.*\/${ins_id}\.//;
+# 		$key =~  s%:%/%g;
+		
+# 		my $uris = $neb->find_instances($key);
+# 		my $found = 0;
+# 		foreach my $uri (@{ $uris }) {
+# 		    if ($uri eq "file://${f}") {
+# 			$found = 1;
+# 		    }
+# 		}
+# 		if ($#{ $uris } == -1) { # true orphan
+# 		    if ($key =~ /ecliptic/) {
+# 			$key =~ s%/00hr%:00hr%;
+# 			$neb->replicate($key);
+# 			print "#replicate $key\n";
+# 		    }
+# 		    $found = -1;
+# 		}
+# 		print "$f $stat[7] $key $found\n";
+# 		if ($found == 0) {
+# #		print "rm $f\n";
+# 		    unlink($f);
+# 		    foreach my $rsync (@rsync_vols) {
+# 			my $test_f = $f;
+# 			$test_f =~ s%^/data/%%;
+# 			$test_f = "/data/${rsync}/stsci_shuffle/${test_f}";
+# 			if (-e $test_f) {
+# 			    unlink($f);
+# #			print "rm $test_f\n";
+# 			}
+# 		    }
+# 		}
+# 	    }
+# 	}
     }
-#    print STDERR "$subdir\n";
 }
+my $Msize = $size / 1024 / 1024;
+print "$root $size $Msize\n";
+print "OTA $size_ota\n";
+print "PV1 $size_pv1\n";
+print "PV2 $size_pv2\n";
+print "MD $size_md\n";
+
Index: trunk/tools/stsci_disks/p1_clear.pl
===================================================================
--- trunk/tools/stsci_disks/p1_clear.pl	(revision 38680)
+++ trunk/tools/stsci_disks/p1_clear.pl	(revision 38682)
@@ -51,4 +51,10 @@
     defined $host && defined $volume && defined $dir && 
     defined $db && defined $dbhost && defined $dbuser && defined $dbpass;
+
+my $host_check = `hostname`;
+chomp($host_check);
+if ($host ne $host_check) {
+    die "Required to run on $host.  not on $host_check\n";
+}
 
 my $dbh = DBI->connect(
@@ -125,5 +131,5 @@
 	    }
 
-	    $md5sums[$i] = local_md5sum($f);
+	    $md5sums[$i] = actual_local_md5sum($f);
 	    unless (defined($md5sums[$i])) {
 		die("Failed to calculate md5sum locally. $f ");
@@ -218,5 +224,8 @@
 	    if (defined($vols[$i])) {
 #		print "HERE: $keys[$i] $host $volume\n";
-		$neb->cull($keys[$i],$host . "." . $volume);
+		eval {
+		    $neb->cull($keys[$i],$host . "." . $volume);
+		};
+		warn "CULL ERROR: $@" if $@;
 	    }
 	    else {
@@ -282,4 +291,15 @@
     };
 
+    if (defined($vol_host)) {
+	if ($vol_host eq 'ipp066') { 
+	    $vol_host = get_volume($key); 
+	    if ($vol_host eq 'ipp066') { 
+		# Really?
+	    	$vol_host = get_volume($key); 
+#		if ($vol_host eq 'ipp066') {}  #fine.  Be that way.  Jerk.
+#                 
+	    }
+	}
+    }
     return($vol_host);
 }
@@ -326,2 +346,13 @@
     return($sum);
 }
+
+sub actual_local_md5sum {
+    my $filename = shift;
+    my $response = `md5sum $filename`;
+    chomp($response);
+    my ($sum, undef) = split /\s+/, $response;
+     unless(defined($sum)) {
+         warn("Failed to calculate md5sum Actually locally. $filename $host $volume");
+     }
+    return($sum);
+}
Index: trunk/tools/stsci_disks/p2_check.pl
===================================================================
--- trunk/tools/stsci_disks/p2_check.pl	(revision 38680)
+++ trunk/tools/stsci_disks/p2_check.pl	(revision 38682)
@@ -327,6 +327,7 @@
     log_print("STOP $stage $stage_id $path_base");
 }
+
 close(MANIFEST);    
-    
+log_print("END $stage $stage_id ${manifest_dir}/${stage}.${stage_id}.MANIFEST");    
 
 sub my_die {
@@ -353,5 +354,5 @@
     my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet);    
     my $max_used_space = 0.98;
-    my $topfew_count = 15;
+    my $topfew_count = 25;
     eval {
 	my $rows;
Index: trunk/tools/stsci_disks/p3_permcheck.pl
===================================================================
--- trunk/tools/stsci_disks/p3_permcheck.pl	(revision 38680)
+++ trunk/tools/stsci_disks/p3_permcheck.pl	(revision 38682)
@@ -71,7 +71,6 @@
 
 my @volumes = ();
-for (my $i = 4; $i <= 4; $i++) {
-    my $max = 3;
-    if ($i == 4) { $max = 2; }
+for (my $i = 4; $i <= 5; $i++) {
+    my $max = 2;
     for (my $j = 0; $j < $max; $j++) {
         push @volumes, sprintf("ippb%02d.%1d",$i,$j);
@@ -83,4 +82,10 @@
 my $command;
 if ($stage eq 'raw') {
+    $command = "regtool -processedimfile -data_state full -exp_id $stage_id -dbname $dbname";
+}
+elsif ($stage eq 'isp') {
+    $command = "regtool -processedimfile -data_state full -exp_id $stage_id -dbname $dbname";
+}
+elsif ($stage eq 'ssp') {
     $command = "regtool -processedimfile -data_state full -exp_id $stage_id -dbname $dbname";
 }
@@ -99,17 +104,17 @@
 my @keys = ();
 foreach my $stageEntry ( @{ $stageData } ) {
-    if ($stage eq 'raw') {
-	if ($stageEntry->{ignored}) { next; }
-	push @keys, $stageEntry->{uri};
+    if (($stage eq 'raw')||($stage eq 'ssp')||($stage eq 'isp')) {
+        if ($stageEntry->{ignored}) { next; }
+        push @keys, $stageEntry->{uri};
     }
     elsif ($stage eq 'cam') {
-	push @keys, $stageEntry->{path_base} . ".smf";
-	for (my $u = 0; $u < 8; $u++) {
-	    for (my $v = 0; $v < 8; $v++) {
-		if ((($u == 0)||($u == 7))&&
-		    (($v == 0)||($v == 7))) { next; }
-		push @keys, $stageEntry->{path_base} . ".XY${u}${v}.mk.fits";
-	    }
-	}
+        push @keys, $stageEntry->{path_base} . ".smf";
+        for (my $u = 0; $u < 8; $u++) {
+            for (my $v = 0; $v < 8; $v++) {
+                if ((($u == 0)||($u == 7))&&
+                    (($v == 0)||($v == 7))) { next; }
+                push @keys, $stageEntry->{path_base} . ".XY${u}${v}.mk.fits";
+            }
+        }
     }
 }
@@ -120,23 +125,23 @@
     my ($ipp_copies,$stsci_copies) = check_instances($key);
     if ($stsci_copies != 0) {
-	for (my $i = 0; $i < $stsci_copies; $i++) {
-	    $neb->replicate($key,$rand_vol);
-	}
-	print ">> REP: $key $rand_vol\n";
-	sleep(5);
-	my ($ipp_copies,$stsci_copies) = check_instances($key);
+        for (my $i = 0; $i < $stsci_copies; $i++) {
+            $neb->replicate($key,$rand_vol);
+        }
+        print ">> REP: $key $rand_vol\n";
+        sleep(5);
+        my ($ipp_copies,$stsci_copies) = check_instances($key);
     }
-    if ($ipp_copies == 1) {
-	$neb->replicate($key,$rand_vol);
-	print ">> REP: $key $rand_vol\n";
-	sleep(5);
-	my ($ipp_copies,$stsci_copies) = check_instances($key);
-    }
-    if ($ipp_copies >= 2) {
-	$good ++;
+#     if ($ipp_copies == 1) {
+#         $neb->replicate($key,$rand_vol);
+#         print ">> REP: $key $rand_vol\n";
+#         sleep(5);
+#         my ($ipp_copies,$stsci_copies) = check_instances($key);
+#     }
+    if ($ipp_copies >= 1) {
+        $good ++;
     }
     else {
-	print ">> $ipp_copies $stsci_copies\n";
-	push @issues, $key;
+        print ">> $ipp_copies $stsci_copies\n";
+        push @issues, $key;
     }
     print "$key $good $#issues\n";
@@ -146,9 +151,9 @@
 if ($#issues != -1) {
     foreach my $key (@issues) {
-	my ($ipp_copies,$stsci_copies) = check_instances($key);
-	if ($ipp_copies < 2) {
-	    print "CHECK: $key\n";
-	    $bad++;
-	}
+        my ($ipp_copies,$stsci_copies) = check_instances($key);
+        if ($ipp_copies < 2) {
+            print "CHECK: $key\n";
+            $bad++;
+        }
     }
 }
@@ -163,11 +168,11 @@
     my $ipp_copies = 0;
     my $stsci_copies = 0;
-    foreach my $uri (@{ $uris }) { 
-	if ($uri =~ /file:...data.ipp/) {
-	    $ipp_copies++;
-	}
-	if ($uri =~ /file:...data.stsci/) {
-	    $stsci_copies++;
-	}
+    foreach my $uri (@{ $uris }) {
+        if ($uri =~ /file:...data.ipp/) {
+            $ipp_copies++;
+        }
+        if ($uri =~ /file:...data.stsci/) {
+            $stsci_copies++;
+        }
     }
     return($ipp_copies,$stsci_copies);