Changeset 37594 for trunk/ippScripts

Timestamp:

Nov 13, 2014, 4:12:26 PM (12 years ago)

Author:

watersc1

Message:

Mostly tested version of the shuffle script with changes to support 1-copy warps correctly.

It'd be best to do a full clean test, but with the stsci nodes still in repair, this would just scatter files to other destinations, so I'll wait to do one final test when those nodes are marked up.

File:

• trunk/ippScripts/scripts/permcheck.pl (modified) (13 diffs)

trunk/ippScripts/scripts/permcheck.pl

@@ -21 +21 @@
 #my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
@@ -41 +42 @@
 pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
     unless $dbname;
-pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
+pod2usage( -msg => "Required options: --stage [camera|warp|stack|skycal]", -exitval => 2 )
     unless $stage;
 pod2usage( -msg => "missing key", exitval => 2 )
@@ -58 +59 @@
                            'ippb06' => 1
 );
+
+if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes.
+    %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1,
+                     'stsci03' => 1, 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
+                     'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1,
+                     'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1,
+                     'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1,
+                     'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1);
+    %backup_destinations = %backup_hosts;
+}
+
 my $backup_Nvols = 3;
 
@@ -113 +125 @@
         &my_die("Unable to parse metadata from stagetool", $stage_id);
 }
+elsif ($stage eq 'warp') {
+    my $cmd = "$warptool -warped -warp_id $stage_id -dbname $dbname";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+        run(command => $cmd, verbose => 0);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stagetool: $error_code", $stage_id);
+    }
+    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from stagetool", $stage_id);
+}
 
 my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
+                  'warp'   => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'],
                   'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
                                'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
@@ -124 +148 @@
     my $data_state = $entry->{state};
     my $hostname = $entry->{hostname};
+    my $quality  = $entry->{quality};
+    if ($quality != 0) { next; }
+    print "# $path_base $data_state $hostname $quality\n";
     foreach my $product (@{ $components{$stage} }) {
         my @keys = ();
@@ -148 +175 @@
             my $stat = $neb->stat($key);
             die "nebulous key: $key not found" unless $stat;   
-            my $instances = $neb->find_instances($key, 'any');
+            my $instances;
+            eval {
+                $instances = $neb->find_instances($key, 'any');
+            };
+            unless (defined($instances)) { print "## skipping due to zero instances\n"; next; }
             die "no instances found" unless $instances;   
             
@@ -256 +287 @@
                     $quality[0] = 1;
                     print "cp $good_file $diskfiles[0]\n";
-                    system("cp $good_file $diskfiles[0]");
+                    vsystem("cp $good_file $diskfiles[0]");
                 }
                 # Begin my best validation thought
@@ -275 +306 @@
                     if ($md5sums[$i] eq $md5sum) { # Found it.
                         print "cp $diskfiles[$i] $diskfiles[0]\n";
-                        system("cp $diskfiles[$i] $diskfiles[0]");
+                        vsystem("cp $diskfiles[$i] $diskfiles[0]");
                         # Begin my best validation thought
                         {
@@ -291 +322 @@
             if ($Ngood == 1) { # We have only one version
                 if ($quality_mask & 1) { # ANd it's on a backup volume
-                    print "neb-replicate $key\n";
-                    if ($do_ops) {
-                        $neb->replicate($key) or die "failed to replicate the single valid copy";
-                        if ($@) { die $@; }
+                    if ($user_copies > 1) { # And we want more than one copy.
+                        print "neb-replicate $key\n";
+                        if ($do_ops) {
+                            $neb->replicate($key) or die "failed to replicate the single valid copy";
+                            if ($@) { die $@; }
+                        }
                     }
                 }
@@ -305 +338 @@
                         
                         # Begin my best validation thought
-                        system("sync") == 0 or die "Couldn't sync?";
+                        vsystem("sync") == 0 or die "Couldn't sync?";
                         my $uris = $neb->find_instances($key,$rep_vol);
                         @$uris = map {URI->new($_)->file if $_} @$uris;
@@ -325 +358 @@
                         
                         # Begin my best validation thought
-                        system("sync") == 0 or die "Couldn't sync?";
+                        vsystem("sync") == 0 or die "Couldn't sync?";
                         my $uris = $neb->find_instances($key,$rep_vol);
                         @$uris = map {URI->new($_)->file if $_} @$uris;
@@ -343 +376 @@
                     }
                 }
-                for (my $i = 0; $i <= $#diskfiles; $i++) {
+            }
+#XXX NEW DEBUG THIS:  This should iterate over diskfiles that are not marked with a quality = 2, and cull them.  
+# For the N>=2, we set that for the primary, and
+# for both cases, we do not have a diskfile entry for the newly replicated copy.  
+# Therefore, this should cull down to the correct number.
+            for (my $i = 0; $i <= $#diskfiles; $i++) {
 #           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-                    if ($quality[$i] != 1) {
-                        if ($existance[$i] == 0) { # This disk file doesn't exist.
-                            system("touch $diskfiles[$i]");
-                        }
-                        my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-                        print "neb-cull --volume $cull_vol $key\n";
-                        if ($do_ops) {
-                            # The tilde here is to force hard volumes.  Don't touch it.
-                            $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-                            if ($@) { die "$@"; }
-                        }
-                    }
-                }
-            }
+                if ($quality[$i] != 1) {
+                    if ($existance[$i] == 0) { # This disk file doesn't exist.
+                        vsystem("touch $diskfiles[$i]");
+                    }
+                    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+                    print "neb-cull --volume $cull_vol $key\n";
+                    if ($do_ops) {
+                        # The tilde here is to force hard volumes.  Don't touch it.
+                        $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+                        if ($@) { die "$@"; }
+                    }
+                }
+            }
+            # }
         } # end keys for this product
     } # end product for this entry
 } # end entry for this id
+
+sub vsystem {
+    my $cmd = shift;
+    print "$cmd\n";
+    if ($do_ops) { 
+        system($cmd);
+    }
+}
 
 sub local_md5sum {
@@ -379 +425 @@
 
 
+
 sub parse_volume {
     my $filename = shift(@_);