Index: branches/eam_branches/ipp-20140423/ippScripts/Build.PL
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/Build.PL	(revision 36693)
+++ branches/eam_branches/ipp-20140423/ippScripts/Build.PL	(revision 36826)
@@ -116,4 +116,5 @@
         scripts/lossy_compress_imfile.pl
         scripts/rawcheck.pl
+        scripts/permcheck.pl
         scripts/ipp_apply_burntool.pl
         scripts/ipp_apply_burntool_single.pl
@@ -134,4 +135,5 @@
         scripts/psphot_fullforce_warp.pl
         scripts/psphot_fullforce_summary.pl
+        scripts/stack_bkg_mk_mdc.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',
Index: branches/eam_branches/ipp-20140423/ippScripts/MANIFEST
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/MANIFEST	(revision 36693)
+++ branches/eam_branches/ipp-20140423/ippScripts/MANIFEST	(revision 36826)
@@ -40,4 +40,5 @@
 scripts/lossy_compress_imfile.pl
 scripts/rawcheck.pl
+scripts/permcheck.pl
 scripts/ipp_apply_burntool.pl
 scripts/ipp_apply_burntool_single.pl
Index: branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl	(revision 36826)
+++ branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl	(revision 36826)
@@ -0,0 +1,414 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings FATAL => qw( all );
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+use Pod::Usage qw( pod2usage );
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Digest::MD5;
+use URI;
+
+my $missing_tools = 0;
+#my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
+
+my ($server,$dbname,$stage,$stage_id);
+
+$server = $ENV{'NEB_SERVER'} unless $server;
+# $dbname = 'gpc1'; 
+GetOptions(
+    'server|s=s'     => \$server,
+    'dbname=s'       => \$dbname,
+    'stage=s'        => \$stage,
+    'stage_id|x=s'     => \$stage_id,
+) || pod2usage( 2 );
+
+# Option parsing
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --server", -exitval => 2 )
+    unless $server;
+pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
+    unless $dbname;
+pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
+    unless $stage;
+pod2usage( -msg => "missing key", exitval => 2 )
+    unless defined $stage_id;
+
+# Global options:
+
+my $do_ops = 1;
+my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
+		    'ippb02' => 1, 'ippb03' => 1,
+		    'ippb04' => 1, 'ippb05' => 1
+    );
+my %backup_destinations = ('ippb04' => 1,
+			   'ippb05' => 1);
+my $backup_Nvols = 3;
+
+my $ipprc;
+if ($dbname eq 'gpc1') {
+    $ipprc = PS::IPP::Config->new( "GPC1" );
+}
+else {
+    die "Unknown camera to use.";
+}
+
+# Set up nebulous db interface
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+);
+die "can't connected to Nebulous Server: $server"
+    unless defined $neb;
+
+# Pull data from the gpc1 database
+my $verbose = 0;
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+my $files;
+if ($stage eq 'camera') {
+    my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+elsif ($stage eq 'stack') {
+    my $cmd = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+elsif ($stage eq 'skycal') {
+    my $cmd = "$staticskytool -skycalresult -skycal_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+	run(command => $cmd, verbose => 0);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from stagetool", $stage_id);
+}
+
+my %components = ('camera' => ['PSASTRO.OUTPUT'],
+		  'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
+			       'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
+		  'skycal' => ['PSASTRO.OUTPUT.CMF']);
+
+
+foreach my $entry (@$files) {
+    my $path_base = $entry->{path_base};
+    my $data_state = $entry->{state};
+    my $hostname = $entry->{hostname};
+    foreach my $product (@{ $components{$stage} }) {
+	my $key = $ipprc->filename($product,$path_base);
+
+# Do validation
+
+	# neb-stat level handling
+	my $stat = $neb->stat($key);
+	die "nebulous key: $key not found" unless $stat;   
+	my $instances = $neb->find_instances($key, 'any');
+	die "no instances found" unless $instances;   
+	
+	my $user_copies;
+	eval {
+	    $user_copies = $neb->getxattr($key, "user.copies");
+	};
+	unless(defined($user_copies)) {
+	    $user_copies = 1;
+	}
+
+	my $md5sum;
+	my @validation;
+	my %md5sum_uniq;
+	my $existing_copies = 0;
+	
+	my @existance;
+	my @md5sums;
+	my @diskfiles = map {URI->new($_)->file if $_} @$instances;
+	my @diskvols;
+	my @diskhosts;
+	my @quality;
+	my $Ngood = 0;
+	my $quality_mask = 0;
+	
+	for (my $i = 0; $i <= $#diskfiles; $i++) {
+	    if (-e $diskfiles[$i]) {
+		$existance[$i] = 1;
+		$existing_copies++;
+		$md5sums[$i] = local_md5sum($diskfiles[$i]);
+		$md5sum_uniq{$md5sums[$i]} = 1;
+	    }
+	    else {
+		$existance[$i] = 0;
+		$md5sums[$i] = 'NON-EXISTANT';
+		$md5sum_uniq{$md5sums[$i]} = 1;
+	    }
+	    ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
+	    $validation[$i] = sprintf("% 3d %32s %s %s %d",
+				      $existance[$i],
+				      $md5sums[$i],
+				      $diskfiles[$i],
+				      $diskhosts[$i],$diskvols[$i]
+	    );
+	    
+	    # Pre-parse decisions
+	    if ($existance[$i] == 0) {
+		$quality[$i] = 0;
+	    }
+	    elsif (is_backup_volume($diskhosts[$i])) {
+		$quality[$i] = 1;
+		$quality_mask = $quality_mask | 1;
+		$Ngood++;
+	    }
+	    else {
+		$quality[$i] = 2;
+		$Ngood++;
+	    }
+	}
+	if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
+	    die "There are multiple md5sum values for $key";
+	}
+	$md5sum = (keys(%md5sum_uniq))[0];
+
+	print "\n$key $data_state $md5sum $hostname\n";
+	if (1) {
+	    
+	    print
+		"object id:             ", @$stat[0], "\n",
+		"key:                   ", @$stat[1], "\n";
+	    print 
+		"epoch:                 ", @$stat[4], "\n";
+	    print 
+		"md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
+	    print 
+		"requested instances:   ", $user_copies, "\n",
+		"available instances:   ", @$stat[6], "\n",
+		"existing instances:    ", $existing_copies, "\n",
+		"total instances:       ", @$stat[7], "\n",
+		"instance location:\n", " " x 4;
+	    print
+		join("\n" . " " x 4, @validation), "\n";
+	}
+
+	# Decide what to do
+	if ($Ngood == 0) {
+	    # DO something to attempt to fix this.
+	    my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
+	    
+	    open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
+	    my $good_file = '';
+	    while (<DD>) {
+		$_ =~ s/^\s+//;
+		my ($z,undef,$ff) = split /\s+/;
+		if (($ff)&&(-e $ff)) {
+		    my $md_response = `md5sum $ff`;
+		    if ($md_response =~ /$md5sum/) {
+			$good_file = (split /\s+/,$md_response)[1];
+		    }
+		}
+	    }
+	    close(DD);
+	    if ($good_file eq '') {
+		die "No valid instance of key: $key";
+	    }
+	    else {
+		$quality[0] = 1;
+		print "cp $good_file $diskfiles[0]\n";
+		system("cp $good_file $diskfiles[0]");
+	    }
+	    # Begin my best validation thought
+	    {
+		my $tmpmd5 = local_md5sum($diskfiles[0]);
+		if ($tmpmd5 ne $md5sum) { 
+		    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		}
+	    }
+	    # End my best validation thought.
+	    
+	    $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
+	}
+    
+	if ($quality[0] == 0) { # The first instance is bad.
+	    # But since we're here, and not up there, there must be at least one good copy.  Find it.
+	    for (my $i = 0; $i <= $#md5sums; $i++) {
+		if ($md5sums[$i] eq $md5sum) { # Found it.
+		    print "cp $diskfiles[$i] $diskfiles[0]\n";
+		    system("cp $diskfiles[$i] $diskfiles[0]");
+		    # Begin my best validation thought
+		    {
+			my $tmpmd5 = local_md5sum($diskfiles[0]);
+			if ($tmpmd5 ne $md5sum) { 
+			    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+			}
+		    }
+		    # End my best validation thought.
+		    last; # We're done here now.
+		}
+	    }	
+	}
+	
+	if ($Ngood == 1) { # We have only one version
+	    if ($quality_mask & 1) { # ANd it's on a backup volume
+		print "neb-replicate $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key) or die "failed to replicate the single valid copy";
+		    if ($@) { die $@; }
+		}
+	    }
+	    else { # And it's not, so put one there
+		my $rep_vol = get_random_backup_volume();
+		print "neb-replicate --volume $rep_vol  $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
+		    if ($@) { die $@; }
+		    
+		    # Begin my best validation thought
+		    system("sync") == 0 or die "Couldn't sync?";
+		    my $uris = $neb->find_instances($key,$rep_vol);
+		    @$uris = map {URI->new($_)->file if $_} @$uris;
+		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
+		    if ($tmpmd5 ne $md5sum) { 
+			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		    }
+		    # End my best validation thought.
+		}	    
+	    }
+	}
+	else { # N >= 2
+	    if (!($quality_mask & 1)) { # no backup copy
+		my $rep_vol = get_random_backup_volume();
+		print "neb-replicate --volume $rep_vol  $key\n";
+		if ($do_ops) {
+		    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
+		    if ($@) { die $@; }
+		    
+		    # Begin my best validation thought
+		    system("sync") == 0 or die "Couldn't sync?";
+		    my $uris = $neb->find_instances($key,$rep_vol);
+		    @$uris = map {URI->new($_)->file if $_} @$uris;
+		    my $tmpmd5 = local_md5sum(${ $uris }[0]);
+		    if ($tmpmd5 ne $md5sum) { 
+			die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+		    }
+		    # End my best validation thought.
+		}	    
+	    }
+	    if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+		for (my $i = 0; $i <= $#diskfiles; $i++) {
+		    if ($quality[$i] == 2) {
+			$quality[$i] = 1;
+			last;
+		    }
+		}
+	    }
+	    for (my $i = 0; $i <= $#diskfiles; $i++) {
+#	    print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
+		if ($quality[$i] != 1) {
+		    if ($existance[$i] == 0) { # This disk file doesn't exist.
+			system("touch $diskfiles[$i]");
+		    }
+		    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+		    print "neb-cull --volume $cull_vol $key\n";
+		    if ($do_ops) {
+			# The tilde here is to force hard volumes.  Don't touch it.
+			$neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+			if ($@) { die "$@"; }
+		    }
+		}
+	    }
+	}
+    } # end product for this entry
+} # end entry for this id
+
+sub local_md5sum {
+    my $filename = shift;
+    my $volume   = (split /\//, $filename)[2];
+    my $host     = $volume;
+    $host =~ s/\.\d//;
+#    print "$filename $host $volume\n";
+    my $response = `ssh $host md5sum $filename`;
+    chomp($response);
+    my ($sum, undef) = split /\s+/, $response;
+    unless(defined($sum)) {
+	my_die("Failed to calculate md5sum locally. $filename $host $volume");
+    }
+    return($sum);
+}
+
+
+sub parse_volume {
+    my $filename = shift(@_);
+    my $full_volume   = (split /\//, $filename)[2];
+    my ($hostname,$vol_index) = split /\./, $full_volume; # /;
+    return($hostname,$vol_index);
+}
+
+sub is_backup_volume {
+    my $hostname = shift(@_);
+    if (exists($backup_hosts{$hostname})) {
+	return(1);
+    }
+    return(0);
+}
+
+sub get_random_backup_volume {
+    my $NN = scalar keys %backup_destinations;
+    my $backup_host = (keys %backup_destinations)[int(rand($NN))];
+    my $backup_vol  = int(rand($backup_Nvols));
+    
+    return("${backup_host}.${backup_vol}");
+}
+
+sub my_die {
+    my $msg = shift(@_);
+    print $msg . "\n";
+    foreach my $a (@_) {
+	print "ARG: $a\n";
+    }
+    die;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+__END__
+
+
Index: branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl	(revision 36693)
+++ branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl	(revision 36826)
@@ -194,5 +194,5 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppSub: $error_code", $ff_id, $warp_id, $skycell_id, $error_code);
+            &my_die("Unable to perform psphotFullForce: $error_code", $ff_id, $warp_id, $skycell_id, $error_code);
         }
 
Index: branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl	(revision 36693)
+++ branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl	(revision 36826)
@@ -14,4 +14,6 @@
 
 my ($seq_id, $tess_id, $projection_cell, $label, $dist_group, $dbname, $pretend, $simple, $verbose, $no_update);
+
+my $good_frac_min = 0.05;
 
 GetOptions(
@@ -87,4 +89,5 @@
     . " -select_skycell_id $projection_cell%"
     . " -set_workdir $workdirBase/$label/$datestr"
+    . " -select_good_frac_min $good_frac_min"
     . " -set_label $label"
     . " -set_data_group $label"
Index: branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl
===================================================================
--- branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl	(revision 36826)
+++ branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl	(revision 36826)
@@ -0,0 +1,91 @@
+#! /usr/bin/env perl
+
+use warnings;
+use strict;
+use Carp;
+use DBI;
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+my ($dbname,$stack_id,$label,$camera,$filter);
+#$dbname = 'gpc1';
+#$camera = 'GPC1';
+GetOptions(
+    'stack_id=s'   => \$stack_id, 
+    'label=s'      => \$label,
+    'filter=s'     => \$filter,
+    'dbname=s'     => \$dbname,
+    'camera=s'     => \$camera,
+    ) or pod2usage ( 2 );
+pod2usage( -msg => "Required options; --stack_id | --label", -exitval => 3) unless
+    ((defined $stack_id)||(defined $label));
+pod2usage( -msg => "Required options; --camera --dbname ", -exitval => 3) unless
+    (defined($camera) && defined($dbname));
+my $verbose = 0;
+
+use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+my $dbserver = 'ippdb01';
+my $dbuser = 'ippuser';
+my $dbpass = 'ippuser';
+my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
+                   "mysql_socket=" . DB_SOCKET(),
+                   ${dbuser},${dbpass}, 
+		   { RaiseError => 1, AutoCommit => 1}
+    ) or die "Unable to connect to database $DBI::errstr\n";
+
+my $ipprc = PS::IPP::Config->new( $camera );
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+
+my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 ";
+
+if (defined($label)) {    $query .= " AND stackRun.label = '${label}' "; }
+if (defined($stack_id)) { $query .= " AND stack_id = ${stack_id} "; }
+if (defined($filter)) {   $query .= " AND stackRun.filter = '${filter}' "; }
+
+my $cams = $db->selectall_arrayref( $query );
+
+#my $logDest = $ipprc->filename($logRule, $outroot, $class_id);
+
+print "backgroundStackInputs MULTI\n\n";
+my $index = 0;
+
+foreach my $cam (@{ $cams }) {
+    my ($chip_id,$cam_id,$cam_path) = @{ $cam };
+    my $smf_file = $ipprc->filename("PSASTRO.OUTPUT",$cam_path);
+    print "backgroundStackInput${index} METADATA\n";
+    print "   chip_id             S64             $chip_id\n";
+    print "   cam_id              S64             $cam_id\n";
+    print "   astrom              STR             $smf_file\n";
+    print "   models              METADATA\n";
+
+    my $command = "chiptool -processedimfile -chip_id $chip_id";
+    $command   .= " -dbname $dbname " if defined $dbname;
+    
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	die "Unable to parse metadata.";
+    }
+    my $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
+	die "Unabel to parse metadata: 2.";
+
+    foreach my $imf (@$imfiles) {
+	my $class_id = $imf->{class_id};
+	my $path_base= $imf->{path_base};
+
+	my $model = $ipprc->filename("PPIMAGE.BACKMDL",$path_base,$class_id);
+
+	print "      ${class_id}            STR           $model\n";
+    }
+    print "   END\n";
+    print "END\n\n";
+    
+    $index++;
+}
+