Index: trunk/tools/stsci_disks/p2_check.pl
===================================================================
--- trunk/tools/stsci_disks/p2_check.pl	(revision 38280)
+++ trunk/tools/stsci_disks/p2_check.pl	(revision 38281)
@@ -33,8 +33,21 @@
 ) || pod2usage(2 );
 
+
+my $server = $ENV{'NEB_SERVER'};
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+    );
+
 my $camera = 'GPC1';
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set_up", $stage, $stage_id);
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $time = time();
+my $logDest = "neb://any/stsci_shuffle/${stage}/${stage}.${stage_id}.log.${time}";
+my $ipprc = PS::IPP::Config->new( "GPC1" ) or &my_die( "Could not create config object.", $stage,$stage_id);
+$ipprc->redirect_output($logDest) or &my_die("Could not redirect output to logfile ${logDest}", $stage,$stage_id);
+
+my $manifest_file = $ipprc->file_resolve("neb://any/stsci_shuffle/${stage}/${stage}.${stage_id}.MANIFEST",1);
+open(MANIFEST,">${manifest_file}") || &my_die( "Could not open manifest file", $stage,$stage_id);
 
 # Define which things we want to look at, and where and how many of each.
@@ -58,5 +71,5 @@
 		   'PPSTACK.UNCONV.EXPWT' => { 'local' => 2, 'stsci' => 1 },
 		   'PSPHOT.PSF.SKY.SAVE' => { 'local' => 2, 'stsci' => 1 },
-		   'PSPHOT.OUT.CMF.MEF' => { 'local' => 2, 'stsci' => 1 },
+		   'PSPHOT.STACK.OUTPUT' => { 'local' => 2, 'stsci' => 1 },
 		   'PPSTACK.CONFIG' => { 'local' => 2, 'stsci' => 1}
 
@@ -85,4 +98,5 @@
 my @keylist = ();
 
+
 my $otime = $time;
 $time = time();
@@ -97,5 +111,23 @@
     
     foreach my $component (keys %components) {
-	my $key = $ipprc->filename($component,$path_base);
+	my $key;
+	if (($stage eq 'stack')&&($component eq 'PSPHOT.STACK.OUTPUT')) {
+	    # Do special things to get to skycalresult
+	    my $ss_command = "$staticskytool -skycalresult -stack_id $stage_id -label LAP.PV3.20140730.skycal01 -dbname $dbname";
+	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		run(command => $command, verbose => $verbose);
+	    unless ($success) {
+		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		&my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
+	    }
+	    my $SS_MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id);
+	    my $SS_data = parse_md_list($MDlist);
+	    my $SSRun = ${ $SS_data }[0];
+	    my $SS_pb = $SSRun->{path_base};
+	    $key = $ipprc->filename($component,$SS_pb);
+	}
+	else {
+	    $key = $ipprc->filename($component,$path_base);
+	}
 
 	# Stealing most of neb-stat here
@@ -117,152 +149,83 @@
 		$md5sum_uniq{$md5sums[$i]} = 1;
 		if ($volume =~ /stsci/) { 
-		    $stsci_count++; 
-		    $stsci_copies[$i] = 1;
-		    $local_copies[$i] = 0;
+		    push @stsci_copies, $files[$i];
 		}
 		elsif ($volume =~ /ipp/) { 
-		    $local_count++; 
-		    $stsci_copies[$i] = 0;
-		    $local_copies[$i] = 1;
+		    push @local_copies, $files[$i];
 		}
 		else { &my_die("You can't exist somewhere not ipp and not stsci",$stage,$stage_id,$files[$i]); }
-		
-	    }
-	}
-	
+	    }
+	}
+	
+	# Check that we have only one md5sum value.
+	if (scalar(keys %md5sum_uniq) != 1) {
+	    &my_die("Key $key has inconsistent md5sum values", $stage,$stage_id);
+	}
+	$md5sum{$key} = (keys %md5sum_uniq)[0];
+
 	# Do the logic at what we found
-
-		
-		
-    
-    
-
-
-
-
-my $server = $ENV{'NEB_SERVER'};
-my $neb = Nebulous::Client->new(
-    proxy => "$server",
-    );
-
-
-foreach my $subdir (@subdirectories) {
-    my @files = <${subdir}/*>;
-    
-    my @belongs;
-    my @keys;
-    my @vols;
-    my @md5sums;
-
-# Pre scan
-    for (my $i = 0; $i <= $#files; $i++) {
-	my $f = $files[$i];
-	my ($ins_id, $belongs);
-	$ins_id = $f;
-	$ins_id =~ s/^$subdir\///;
-	$ins_id =~ s/\..*//;
-
-	$belongs[$i] = 0; 
-
-	if ($ins_id > 4956576912) { 
-	    if ($f =~ /LAP.PV3.20140730/) { # Only PV3
-		if (($f =~ /wrp/)) {
-		    if (($f =~ /fits/)||($f =~ /cmf/)||($f =~ /psf/)) {
-			$belongs[$i] = 1;
-		    }
-		}
-		if (($f =~ /stk/)&&($f =~ /fits/)&&($f =~ /unconv/)) {
-		    $belongs[$i] = 1;
-		}
-	    }
-	}
-	
-	if ($belongs[$i] == 0) {
-	    $keys[$i] = $f;
-	    $keys[$i] =~ s/.*\/${ins_id}\.//;
-	    $keys[$i] =~ s%:%/%g;
-	    # Compressed fits 
-	    if ($keys[$i] =~ /.fits.fz$/) { 
-		$keys[$i] =~ s/.fits.fz$/.fits/;
-	    }
-
-	    $md5sums[$i] = local_md5sum($f);
-	    # Get a target
-	    $vols[$i] = get_volume($keys[$i]);
-	    if (defined($vols[$i])) {
-		$vols[$i] = $vols[$i] . ".0";
-	    }
-	}
-	elsif ($belongs[$i] == 1) {
-	    $md5sums[$i] = 'NOT_CALCULATED';
-	    $vols[$i]    = 'ippXXX.Y';
-	    $keys[$i]    = 'NO_REPLICATE';
-	}
-	if (defined($vols[$i])) {
-	    print "## $ins_id $md5sums[$i] $vols[$i] $belongs[$i] $f $keys[$i] \n";
-	}
-	else {
-	    print "## $ins_id $md5sums[$i] NOVOL_BECAUSE_UNKNOWN $belongs[$i] $f $keys[$i] \n";
-	}
-    }
+	if ($#stsci_copies + 1 < $components{$component}{stsci}) {
+	    # We need to make an stsci copy.
+	    my $vol = choose_random_stsci();
+	    push @{ $replicates{$key} }, $vol;
+	}
+	if ($#local_copies + 1 < $components{$component}{local}) {
+	    my $vol = get_volume($key);
+	    while ($vol =~ /stsci/) {
+		$vol = get_volume($key);
+	    }
+	    push @{ $replicates{$key} }, $vol;
+	}
+	if ($#stsci_copies + 1 > $components{$component}{stsci}) {
+	    for (my $i = $components{$component}{stsci} + 1; $i <= $#stsci_copies; $i++) {
+		push @{ $culls{$key} }, $stsci_copies[$i];
+	    }
+	}
+	if ($#local_copies + 1 > $components{$component}{stsci}) {
+	    for (my $i = $components{$component}{local} + 1; $i <= $#local_copies; $i++) {
+		push @{ $culls{$key} }, $local_copies[$i];
+	    }
+	}
+    }
+    log_print("SCAN $stage $stage_id $path_base");
+
+    foreach my $key (keys %replicates) {
+	foreach my $vol (@{ $replicates{$key} }) {
+	    $neb->replicate($key,$vol) or &my_die "failed to replicate a copy of $key to $vol";
+	    if ($@) { die $@; }
+	}
+    }
+    system("sync") == 0 or die "Couldn't sync?";
+    log_print("REPL $stage $stage_id $path_base");
+
+    foreach my $key (keys %replicates) {
+	foreach my $vol (@{ $replicates{$key} }) {
+	    my $uris = $neb->find_instances($key,$vol);
+	    @$uris = map {URI->new($_)->file if $_} @$uris;
+	    $uri = ${ $uris }[0];
+	    my $tmpmd5 = local_md5sum($uri);
+	    if ($tmpmd5 ne $md5sums{$key}) {
+		&my_die "Post-replication md5sum does not match for $key on $vol! $tmpmd5 != $md5sums{$key}";
+	    }
+	}
+    }
+    log_print("CHECK $stage $stage_id $path_base");
+    foreach my $key (keys %culls) {
+	foreach my $vol (@{ $culls{$key} }) {
+	    $neb->cull($key,$vol);
+	}
+    }
+    log_print("CULL $stage $stage_id $path_base");
+}
+    
+    
+
+	
+sub log_print {
+    my $msg = shift;
     $otime = $time;
     $time = time();
     $dtime = $time - $otime;
-    print "LOG SCAN $subdir $time $dtime\n";
-#    die;
-# Do a replication pass
-    for (my $i = 0; $i <= $#files; $i++) {
-	if ($belongs[$i] == 0) {
-	    if (defined($vols[$i])) {
-		$neb->replicate($keys[$i],$vols[$i]) or die "failed to replicate a copy to the backup node";
-		if ($@) { die $@; }
-	    }
-	    else {
-		# Key wasn't defiend
-		my $dir = dirname($keys[$i]);
-		system("mkdir -p ${unknown_dir}/${dir}");
-		system("cp -a $files[$i] ${unknown_dir}/$keys[$i]");
-	    }
-	    system("sync") == 0 or die "Couldn't sync?";
-	}
-    }
-    $otime = $time;
-    $time = time();
-    $dtime = $time - $otime;
-    print "LOG REPL $subdir $time $dtime\n";
-
-# Do the check pass
-    for (my $i = 0; $i <= $#files; $i++) {
-	my $uri;
-	if ($belongs[$i] == 0) {
-	    if (defined($vols[$i])) {
-		my $uris = $neb->find_instances($keys[$i],$vols[$i]);
-		@$uris = map {URI->new($_)->file if $_} @$uris;
-		$uri = ${ $uris }[0];
-	    }
-	    else {
-		$uri = "${unknown_dir}/$keys[$i]";
-	    }
-
-	    my $tmpmd5 = local_md5sum($uri);
-	    if ($tmpmd5 ne $md5sums[$i]) { 
-		die "Post-replication md5sum does not match! $tmpmd5 != $md5sums[$i]";
-	    }
-	    # End my best validation thought.	    
-
-	    if (defined($vols[$i])) {
-		$neb->cull($keys[$i],$host . "." . $volume);
-	    }
-	    else {
-		system("rm $files[$i]");
-	    }		
-	}
-    }
-
-    $otime = $time;
-    $time = time();
-    $dtime = $time - $otime;
-    print "LOG END $subdir $time $dtime\n";
-    die;
+    print "LOG $msg $time $dtime\n";
 }
 
@@ -319,30 +282,4 @@
 }
     
-
-sub secure_replicull {
-    my $key = shift;
-    my $md5sum = shift;
-    my $rep_vol = shift;
-    my $del_vol = shift;
-
-    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-    if ($@) { die $@; }
-
-    # Begin my best validation thought
-    system("sync") == 0 or die "Couldn't sync?";
-    my $uris = $neb->find_instances($key,$rep_vol);
-    @$uris = map {URI->new($_)->file if $_} @$uris;
-    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-    if ($tmpmd5 ne $md5sum) { 
-	sleep(2);
-	$tmpmd5 = local_md5sum(${ $uris }[0]);
-	if ($tmpmd5 ne $md5sum) { 
-	    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-	}
-    }
-    # End my best validation thought.
-    
-    $neb->cull($key,$del_vol);
-}
 
 sub local_md5sum {