Changeset 37804 for branches/eam_branches/ipp-20140904/ippScripts/scripts/permcheck.pl

Timestamp:

Jan 11, 2015, 2:05:35 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20140904

Files:

• . (modified) (1 prop)
• ippScripts/scripts/permcheck.pl (modified) (13 diffs)

branches/eam_branches/ipp-20140904/ippScripts/scripts/permcheck.pl

@@ -21 +21 @@
 #my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
@@ -41 +42 @@
 pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
     unless $dbname;
-pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
+pod2usage( -msg => "Required options: --stage [camera|warp|stack|skycal]", -exitval => 2 )
     unless $stage;
 pod2usage( -msg => "missing key", exitval => 2 )
@@ -58 +59 @@
                            'ippb06' => 1
 );
+
+if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes.
+    %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1,
+                     'stsci03' => 1, 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
+                     'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1,
+                     'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1,
+                     'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1,
+                     'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1);
+    %backup_destinations = %backup_hosts;
+}
+
 my $backup_Nvols = 3;
 
@@ -113 +125 @@
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
+elsif ($stage eq 'warp') {
+    my $cmd = "$warptool -warped -warp_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+        run(command => $cmd, verbose => 0);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from stagetool", $stage_id);
+}
 
 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
+                  'warp'   => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'],
                   'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
                                'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
@@ -124 +148 @@
     my $data_state = $entry->{state};
     my $hostname = $entry->{hostname};
+    my $quality  = $entry->{quality};
+    if ($quality != 0) { next; }
+    print "# $path_base $data_state $hostname $quality\n";
     foreach my $product (@{ $components{$stage} }) {
         my @keys = ();
@@ -148 +175 @@
             my $stat = $neb->stat($key);
             die "nebulous key: $key not found" unless $stat;   
-            my $instances = $neb->find_instances($key, 'any');
+            my $instances;
+            eval {
+                $instances = $neb->find_instances($key, 'any');
+            };
+            unless (defined($instances)) { print "## skipping due to zero instances\n"; next; }
             die "no instances found" unless $instances;   
             
@@ -256 +287 @@
                     $quality[0] = 1;
                     print "cp $good_file $diskfiles[0]\n";
-                    system("cp $good_file $diskfiles[0]");
+                    vsystem("cp $good_file $diskfiles[0]");
                 }
                 # Begin my best validation thought
@@ -275 +306 @@
                     if ($md5sums[$i] eq $md5sum) { # Found it.
                         print "cp $diskfiles[$i] $diskfiles[0]\n";
-                        system("cp $diskfiles[$i] $diskfiles[0]");
+                        vsystem("cp $diskfiles[$i] $diskfiles[0]");
                         # Begin my best validation thought
                         {
@@ -291 +322 @@
             if ($Ngood == 1) { # We have only one version
                 if ($quality_mask & 1) { # ANd it's on a backup volume
-                    print "neb-replicate $key\n";
-                    if ($do_ops) {
-                        $neb->replicate($key) or die "failed to replicate the single valid copy";
-                        if ($@) { die $@; }
+                    if ($user_copies > 1) { # And we want more than one copy.
+                        print "neb-replicate $key\n";
+                        if ($do_ops) {
+                            $neb->replicate($key) or die "failed to replicate the single valid copy";
+                            if ($@) { die $@; }
+                        }
                     }
                 }
@@ -305 +338 @@
                         
                         # Begin my best validation thought
-                        system("sync") == 0 or die "Couldn't sync?";
+                        vsystem("sync") == 0 or die "Couldn't sync?";
                         my $uris = $neb->find_instances($key,$rep_vol);
                         @$uris = map {URI->new($_)->file if $_} @$uris;
@@ -325 +358 @@
                         
                         # Begin my best validation thought
-                        system("sync") == 0 or die "Couldn't sync?";
+                        vsystem("sync") == 0 or die "Couldn't sync?";
                         my $uris = $neb->find_instances($key,$rep_vol);
                         @$uris = map {URI->new($_)->file if $_} @$uris;
@@ -343 +376 @@
                     }
                 }
-                for (my $i = 0; $i <= $#diskfiles; $i++) {
+            }
+#XXX NEW DEBUG THIS:  This should iterate over diskfiles that are not marked with a quality = 2, and cull them.  
+# For the N>=2, we set that for the primary, and
+# for both cases, we do not have a diskfile entry for the newly replicated copy.  
+# Therefore, this should cull down to the correct number.
+            for (my $i = 0; $i <= $#diskfiles; $i++) {
 #           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-                    if ($quality[$i] != 1) {
-                        if ($existance[$i] == 0) { # This disk file doesn't exist.
-                            system("touch $diskfiles[$i]");
-                        }
-                        my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-                        print "neb-cull --volume $cull_vol $key\n";
-                        if ($do_ops) {
-                            # The tilde here is to force hard volumes.  Don't touch it.
-                            $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-                            if ($@) { die "$@"; }
-                        }
-                    }
-                }
-            }
+                if ($quality[$i] != 1) {
+                    if ($existance[$i] == 0) { # This disk file doesn't exist.
+                        vsystem("touch $diskfiles[$i]");
+                    }
+                    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+                    print "neb-cull --volume $cull_vol $key\n";
+                    if ($do_ops) {
+                        # The tilde here is to force hard volumes.  Don't touch it.
+                        $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+                        if ($@) { die "$@"; }
+                    }
+                }
+            }
+            # }
         } # end keys for this product
     } # end product for this entry
 } # end entry for this id
+
+sub vsystem {
+    my $cmd = shift;
+    print "$cmd\n";
+    if ($do_ops) { 
+        system($cmd);
+    }
+}
 
 sub local_md5sum {
@@ -379 +425 @@
 
 
+
 sub parse_volume {
     my $filename = shift(@_);