Changeset 27795 for trunk/pstamp/scripts

Timestamp:

Apr 28, 2010, 2:18:01 PM (16 years ago)

Author:

bills

Message:

first crack at queuing automatic updates for diffSkyfiles. Various bug fixes found in testing

Location:

trunk/pstamp/scripts

Files:

• pstamp_checkdependent.pl (modified) (6 diffs)
• pstampparse.pl (modified) (3 diffs)

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -270 +270 @@
 
         if ($good_to_go) {
+            # the reason we defer setting the warp to update is so that we can handle error conditions at previous
+            # stages more easily.
             my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
             $command .= " -set_label $rlabel" if $rlabel;
@@ -341 +343 @@
         my $skycell_id = $skycell->{skycell_id};
 
-        # put this in a module somewhere
+        # XXX: put this in a module somewhere
         my $IPP_DIFF_MODE_WARP_WARP   = 1;
         my $IPP_DIFF_MODE_WARP_STACK  = 2;
@@ -347 +349 @@
         my $IPP_DIFF_MODE_STACK_STACK = 4;
         if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
-            # check the state of the stack
+            # check the state of the template stack
             my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}";
             my $stack = runToolAndParseExpectOne($command, $verbose);
@@ -361 +363 @@
             # now check the warp
             $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
-            my $warp = runToolAndExpectOne($command, $verbose);
-            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
-                if !$warp;
-        }
-
-        my $good_to_go = 1;
-        my @chipsToUpdate;
-        my $chip_id;
-        my $warp_id;
-        my $data ;  #XXXX
-        foreach my $chip (@$data) {
-            $chip_id = $chip->{chip_id};
-            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
-                $good_to_go = 0;
-                push @chipsToUpdate, $chip;
-            } else {
-                # this chip is good to go
-            }
-        }
-
-        if ($good_to_go) {
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            my $warp = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
+
+            if ($warp->{data_state} ne 'full') {
+                return queue_update_warp($warp, 0, $rlabel, $need_magic);
+            }
+            # warps are ready fall through and queue the diff update
+        } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
+            my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
+            my $warp1 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
+
+            if ($warp1->{data_state} ne 'full') {
+                my $status = queue_update_warp($warp1, 0, $rlabel, $need_magic);
+                if ($status) {
+                    return $status;
                 }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif (scalar @chipsToUpdate > 0) {
-            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
-        }
-    } else {
-        my $warpRun = $metadata;
-        my $warp_id = $warpRun->{warp_id};
-        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
-        my $data = runToolAndParse($command, $verbose);
-        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
-            if !$data or scalar @$data != 1;
-
-        my $chipRun = $data->[0];
-
-        my $chipRunState = $chipRun->{state};
-        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
-            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
-            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
-            return 0;
-        }
-        my $warpRunState = $warpRun->{state};
-        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
-            # The inputs and outputs are ready. Queue the warpRun for update.
-
-            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
-                }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif ($chipRunState eq 'cleaned' or 
-            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
-            my $data = [$chipRun];
-            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
-        }
-    }
-
+            }
+            $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
+            my $warp2 = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
+
+            if ($warp2->{data_state} ne 'full') {
+                return queue_update_warp($warp2, 0, $rlabel, $need_magic);
+            }
+            # fall through to queue the diff update
+        } else {
+            # XXX: check stack stack
+            print STDERR "unexpected diff_mode found: $diff_mode\n";
+            return $PS_EXIT_PROG_ERROR;
+        }
+
+        my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
+        $command .= " -set_label $rlabel" if $rlabel;
+
+        if (!$no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+            unless ($success) {
+                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping $command\n";
+        }
+    } else {
+        # XXX todo whole_run
+        return $PS_EXIT_PROG_ERROR;
+    }
     # return value may be the return status of script so zero is good
     return 0;
@@ -593 +567 @@
     carp $msg;
 
-    my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatedependent -fault $fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -600 +574 @@
                     run(command => $command, verbose => $verbose);
         unless ($success) {
-            my_die("failed to set pstampDependent.fault for dep_id: $dep_id",
-                $PS_EXIT_UNKNOWN_ERROR);
+            carp "$cmd failed";
         }
     } else {

trunk/pstamp/scripts/pstampparse.pl

@@ -536 +536 @@
     # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
     if ($stage ne 'raw') {
-        # not ready to handle diff update yet. May never support stack
-        my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff'));
+        # updates for stack stage not supported yet
+        my $allow_wait_for_update = ($stage ne 'stack');
         my $run_state = $image->{state};
         my $data_state = $image->{data_state};
@@ -548 +548 @@
             $newState = 'stop';
             $fault = $PSTAMP_GONE;
-        } elsif ($allow_wait_for_update and 
-                (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) {
+        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
             if ($stage eq 'chip') {
                 my $burntool_state = $image->{burntool_state};
@@ -556 +555 @@
                     $fault = $PSTAMP_NOT_AVAILABLE;
                 }
+            } 
+            if (!$allow_wait_for_update) {
+                print STDERR "wait for update not supported for stage $stage yet\n";
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
             }
             if (!$fault) {