Changeset 24038 for trunk/ippTasks

Timestamp:

May 1, 2009, 5:28:52 PM (17 years ago)

Author:

bills

Message:

on receive side of distribution, optionally create a fileset in a status data store
defer {$stage}tool -importrun until all of the files have been downloaded
in the dist.queuruns task queue rcruns

Location:

trunk/ippTasks

Files:

• dist.pro (modified) (3 diffs)
• rcserver.pro (modified) (1 diff)
• receive.pro (modified) (10 diffs)

trunk/ippTasks/dist.pro

@@ -42 +42 @@
     active true
   end
+  task dist.queueruns
+    active true
+  end
 end
 macro dist.off
@@ -54 +57 @@
   end
   task dist.advance.run
+    active false
+  end
+  task dist.queueruns
     active false
   end
@@ -298 +304 @@
 
   periods      -poll $RUNPOLL
-  periods      -exec 10
+  periods      -exec 30
   periods      -timeout 45
   npending     1

trunk/ippTasks/rcserver.pro

@@ -173 +173 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
+#  periods      -exec $LOADEXEC
+  periods      -exec 20
   periods      -timeout 30
   npending     1

trunk/ippTasks/receive.pro

@@ -20 +20 @@
 ### 
 ### receivetool -revert: remove result after something goes wrong
+###
+### receivetool -toadvance: filesets for which all files have been downloaded and are ready
+### to complete processing
 
 # test for required global variables
@@ -27 +30 @@
 book init receiveFileset
 book init receiveFile
+book init receiveAdvance
 
 macro receive.status
@@ -32 +36 @@
   book listbook receiveFileset
   book listbook receiveFile
+  book listbook receiveAdvance
 end
 
@@ -38 +43 @@
   book init receiveFileset
   book init receiveFile
+  book init receiveAdvance
 end
 
@@ -59 +65 @@
     active true
   end
+  task receive.advance.load
+    active true
+  end
+  task receive.advance.run
+    active true
+  end
 end
 
@@ -80 +92 @@
     active false
   end
+  task receive.advance.load
+    active false
+  end
+  task receive.advance.run
+    active false
+  end
 end
 
@@ -85 +103 @@
 # this variable will cycle through the known database names
 $receive_DB = 0
+$receive_Advance_DB = 0
 
 task           receive.source.load
@@ -380 +399 @@
     book getword receiveFile $pageName product -var PRODUCT
     book getword receiveFile $pageName fileset -var FILESET
+    book getword receiveFile $pageName fileset_id -var FILESET_ID
     book getword receiveFile $pageName file -var FILE
     book getword receiveFile $pageName workdir -var WORKDIR
@@ -388 +408 @@
     stderr $LOGDIR/receive.file.log
 
-    $run = receive_file.pl --file_id $FILE_ID --source $SOURCE --product $PRODUCT --fileset $FILESET --file $FILE --workdir $WORKDIR
+    $run = receive_file.pl --file_id $FILE_ID --source $SOURCE --product $PRODUCT --fileset $FILESET --fileset_id $FILESET_ID --file $FILE --workdir $WORKDIR
     add_standard_args run
 
@@ -419 +439 @@
   end
 end
+
+task           receive.advance.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/receive.advance.log
+
+  task.exec
+    $run = receivetool -toadvance
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$receive_Advance_DB
+      $run = $run -dbname $DB:$receive_Advance_DB
+      $receive_Advance_DB ++
+      if ($receive_Advance_DB >= $DB:n) set receive_Advance_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout receiveAdvance -key fileset_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook receiveAdvance
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup receiveAdvance
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task           receive.advance.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    book npages receiveAdvance -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    book getpage receiveAdvance 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword receiveAdvance $pageName pantaskState RUN
+    book getword receiveAdvance $pageName fileset_id -var FILESET_ID
+    book getword receiveAdvance $pageName fileset -var FILESET
+    book getword receiveAdvance $pageName dbinfo_uri -var DBINFO_URI
+    book getword receiveAdvance $pageName status_product -var STATUS_PRODUCT
+    book getword receiveAdvance $pageName ds_dbname -var DS_DBNAME
+    book getword receiveAdvance $pageName ds_dbhost -var DS_DBHOST
+    book getword receiveAdvance $pageName dbname -var DBNAME
+
+    stdout $LOGDIR/receive.advance.log
+    stderr $LOGDIR/receive.advance.log
+
+    $run = receive_advance.pl --fileset_id $FILESET_ID --fileset $FILESET --dbinfo_uri $DBINFO_URI --status_product $STATUS_PRODUCT --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit receiveAdvance $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword receiveAdvance $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword receiveAdvance $options:0 pantaskState TIMEOUT
+  end
+end