Changeset 23497 for trunk/ippTasks

Timestamp:

Mar 24, 2009, 10:18:19 AM (17 years ago)

Author:

bills

Message:

advance exposures using load, run style so that the proper workdir
and end stage

File:

• trunk/ippTasks/summit.copy.pro (modified) (6 diffs)

trunk/ippTasks/summit.copy.pro

@@ -27 +27 @@
 # list of summit imfiles that need to be downloaded
 book init pzPendingImfile
+# list of pzDownloadExps that have completed downloading
+book init pzPendingAdvance
 
 macro copy.on
@@ -50 +52 @@
       active true
   end
+  task summit.toadvance
+      active true
+  end
   task summit.advance
       active true
@@ -75 +80 @@
   end
   task pztool.clearfault
+      active false
+  end
+  task summit.toadvance
       active false
   end
@@ -86 +94 @@
 $pztoolPendingExp_DB = 0
 $pztoolPendingImfile_DB = 0
+$pztoolPendingAdvance_DB = 0
 $pztoolClearFault_DB = 0
 $pztoolAdvance_DB = 0;
@@ -433 +442 @@
         stderr $LOGDIR/summit.copy.log
 
-        $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose --copies 2
+        $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 120 --verbose --copies 2
         if ($COMPRESS) 
             $run = $run --compress
@@ -510 +519 @@
 end
 
-# promote exposures for which all imfiles have been copied
-task           summit.advance
-  host         local
-
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-
-  stdout NULL
-  stderr $LOGDIR/summit.advance.log
-
-  task.exec
-    $run = pztool -advance
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$pztoolAdvance_DB
-      $run = $run -dbname $DB:$pztoolAdvance_DB
-      $pztoolAdvance_DB ++
-      if ($pztoolAdvance_DB >= $DB:n) set pztoolAdvance_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-
-  # success
-  task.exit    0
-  end
-
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-
-  task.exit    crash
-    showcommand crash
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-
+# build a book of exposures that have completed downloading
+task summit.toadvance
+    host         local
+
+    periods      -exec     30
+    periods      -poll      1
+    periods      -timeout  120
+    # trage       16:00 23:59
+    # trage       00:00 04:00
+    npending     1
+
+    # select entries from the current DB; cycle to the next DB, if it exists
+    # iff the DB list is not set, use the value defined in .ipprc
+    task.exec
+      if ($DB:n == 0)
+        option DEFAULT
+        command pztool -toadvance -limit 40
+      else
+        # save the DB name for the exit tasks
+        option $DB:$pztoolPendingAdvance_DB
+        command pztool -toadvance -limit 60 -dbname $DB:$pztoolPendingAdvance_DB
+        $pztoolPendingAdvance_DB ++
+        if ($pztoolPendingAdvance_DB >= $DB:n) set pztoolPendingAdvance_DB = 0
+      end
+    end
+  
+    # success
+    task.exit    0
+        # convert 'stdout' to book format
+        ipptool2book stdout pzPendingAdvance -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT
+        book shuffle pzPendingAdvance 
+
+        # delete existing entries in the appropriate pantaskStates
+        process_cleanup pzPendingAdvance
+    end
+
+    task.exit     default
+        showcommand failure
+    end
+    task.exit     crash
+        showcommand crash
+    end
+    task.exit     timeout
+        showcommand timeout
+    end
+end
+
+task summit.advance
+    periods      -exec     5
+    periods      -poll     0.05
+    periods      -timeout  650
+    # trage       16:00 23:59
+    # trage       00:00 04:00
+
+    task.exec
+        if ($NETWORK == 0) break
+
+        # if we are waiting on data, make the interval long
+        book npages pzPendingAdvance -var N
+        if ($N == 0)
+            periods -exec 20
+            break
+        end
+        periods -exec 0.05
+
+        # find an exp that needs imfiles fetched
+        book getpage pzPendingAdvance 0 -var pageName -key pantaskState INIT
+        if ("$pageName" == "NULL") break
+
+        # set that exp to run
+        book setword pzPendingAdvance $pageName pantaskState RUN
+
+        book getword pzPendingAdvance $pageName exp_name  -var EXP_NAME
+        book getword pzPendingAdvance $pageName camera    -var CAMERA
+        book getword pzPendingAdvance $pageName telescope -var TELESCOPE
+        book getword pzPendingAdvance $pageName dbname    -var DBNAME
+
+        # 2007-08-30T05:09:59Z
+        substr $DATEOBS 0 4 YEAR
+        substr $DATEOBS 5 2 MONTH
+        substr $DATEOBS 8 2 DAY
+
+        # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship
+        # this function uses workdir_template, default_host, volume_template, volume_default,
+        # it sets workdir and volume
+        set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base
+
+        $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY
+
+        # workdir examples:
+        # file://data/@HOST@.0/gpc1/20080130
+        # neb://@HOST@.0/gpc1/20080130
+
+        stdout $LOGDIR/summit.advance.log
+        stderr $LOGDIR/summit.advance.log
+
+        $run = pztool -advance -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -end_stage reg -workdir $workdir -dbname $DBNAME
+
+        # store the pageName for future reference below
+        options $pageName
+
+        # create the command line
+        if ($VERBOSE > 1)
+          echo command $run
+        end
+        command $run
+    end
+
+    # default exit status
+    task.exit default
+        process_exit pzPendingAdvance $options:0 $JOB_STATUS
+    end
+
+    task.exit crash
+        showcommand crash
+        book setword pzPendingAdvance $options:0 pantaskState CRASH
+    end 
+
+    # operation timed out?
+    task.exit timeout
+        showcommand timeout
+        book setword pzPendingAdvance $options:0 pantaskState TIMEOUT
+    end 
+end