Changeset 28375 for trunk/ippTasks

Timestamp:

Jun 16, 2010, 5:30:27 PM (16 years ago)

Author:

watersc1

Message:

merge of stack association and warp/stack/diff jpegs

Location:

trunk

Files:

• . (modified) (1 prop)
• ippTasks/diff.pro (modified) (7 diffs)
• ippTasks/stack.pro (modified) (7 diffs)
• ippTasks/warp.pro (modified) (6 diffs)

trunk/ippTasks/diff.pro

@@ -11 +11 @@
 book init diffSkyfile
 #book init diffCleanup
+book init diffPendingSummary
 
 ### Database lists
@@ -16 +17 @@
 $diffAdvance_DB = 0
 $diff_revert_DB = 0
+$diffSummary_DB = 0
 #$diffCleanup_DB = 0
 
@@ -27 +29 @@
 macro diff.reset
   book init diffSkyfile
+  book init diffPendingSummary
 #  book init diffCleanup
 end
@@ -44 +47 @@
     active false
   end
+  task diff.summary.load
+    active true
+  end
+  task diff.summary.run
+    active true
+  end
 end
 
@@ -58 +67 @@
   end
   task diff.revert
+    active false
+  end
+  task diff.summary.load
+    active false
+  end
+  task diff.summary.run
     active false
   end
@@ -193 +208 @@
     # host anyhost
     # $WORKDIR = $WORKDIR_TEMPLATE
-
     if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) 
         $DIFF_TAG = ""
@@ -477 +491 @@
   end
 end
+
+### Load tasks for doing the diffs
+### Tasks are loaded into diffPendingSkyCell.
+task           diff.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/diff.summary.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = difftool -tosummary
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$diffSummary_DB
+      $run = $run -dbname $DB:$diffSummary_DB
+      $diffSummary_DB ++
+      if ($diffSummary_DB >= $DB:n) set diffSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    # XXX change tess_id to tess_dir when db is updated
+    ipptool2book stdout diffPendingSummary -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook diffPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup diffPendingSummary
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for calculating the diff overlaps
+### Tasks are taken from diffPendingSkyCell.
+task           diff.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages diffPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in diffPendingSkyCell (pantaskState == INIT)
+    book getpage diffPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword diffPendingSummary $pageName pantaskState RUN
+    book getword diffPendingSummary $pageName diff_id -var DIFF_ID
+    book getword diffPendingSummary $pageName camera -var CAMERA
+    book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword diffPendingSummary $pageName dbname -var DBNAME
+    book getword diffPendingSummary $pageName tess_id -var TESS_DIR
+    book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE
+    book getword diffPendingSummary $pageName state -var RUN_STATE
+
+    # set the host and workdir based on the skycell hash
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+#    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+#    if ("$PATH_BASE" == "NULL") 
+
+    if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) 
+        $DIFF_TAG = ""
+    end
+    if ($DIFF_MODE == 2)
+        $DIFF_TAG = "WS."
+    end 
+    if ($DIFF_MODE == 3)
+        $DIFF_TAG = "SW."
+    end
+    if ($DIFF_MODE == 4)
+        $DIFF_TAG = "SS."
+    end
+
+    basename $TESS_DIR -var TESS_ID
+
+    ## generate outroot specific to this exposure
+    sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID
+
+
+    stdout $LOGDIR/diff.summary.log
+    stderr $LOGDIR/diff.summary.log
+
+    $run = skycell_jpeg.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit diffPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword diffPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword diffPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTasks/stack.pro

@@ -10 +10 @@
 ### Initialise the books containing the tasks to do
 book init stackSumSkyfile
+book init stackPendingSummary
 #book init stackCleanup
 
@@ -16 +17 @@
 $stack_revert_DB = 0
 #$stackCleanup_DB = 0
+$stackSummary_DB = 0
 
 ### Check status of stacking tasks
@@ -26 +28 @@
 macro stack.reset
   book init stackSumSkyfile
+  book init stackPendingSummary
 #  book init stackCleanup
 end
@@ -40 +43 @@
     active false
   end
+  task stack.summary.load
+    active true
+  end
+  task stack.summary.run
+    active true
+  end
 end
 
@@ -51 +60 @@
   end
   task stack.revert
+    active false
+  end
+  task stack.summary.load
+    active false
+  end
+  task stack.summary.run
     active false
   end
@@ -349 +364 @@
 # end
 
+
+
+
 task stack.revert
   host         local
@@ -400 +418 @@
 end
 
+### Load tasks for doing the stack summary
+### Tasks are loaded into stackPendingSummary
+task           stack.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  trange       07:00:00 08:00:00 -nmax 1
+  npending     1
+
+  stdout       NULL
+  stderr       $LOGDIR/stack.summary.load
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = stacktool -tosummary
+    if ($DB:n == 0) 
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$stackSummary_DB
+      $run = $run -dbname $DB:$stackSummary_DB
+      $stackSummary_DB ++
+      if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit     0
+    # convert 'stdout' to book format
+    ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook stackPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup stackPendingSummary
+  end
+
+  # locked list
+  task.exit     default
+    showcommand failure
+  end
+
+  task.exit     crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit     timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for doing the stack summary
+task           stack.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages stackPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+
+    # look for new images in stackPendingSummary (pantaskState == INIT)
+    book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword stackPendingSummary $pageName pantaskState RUN
+    book getword stackPendingSummary $pageName sass_id -var SASS_ID
+    book getword stackPendingSummary $pageName camera -var CAMERA
+    book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword stackPendingSummary $pageName dbname -var DBNAME
+    book getword stackPendingSummary $pageName tess_id -var TESS_DIR
+    book getword stackPendingSummary $pageName state -var RUN_STATE
+    book getword stackPendingSummary $pageName projection_cell -var projection_cell
+
+    # set the host and workdir
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+
+    basename $TESS_DIR -var TESS_ID
+
+    ## generate outroot specific to this association
+    sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID
+
+    stdout $LOGDIR/stack.summary.log
+    stderr $LOGDIR/stack.summary.log
+
+    $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+        echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit       default
+    process_exit stackPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit       crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword stackPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit       timeout
+    showcommand timeout
+    book setword stackPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end
+

trunk/ippTasks/warp.pro

@@ -13 +13 @@
 book init warpPendingSkyCell
 book init warpPendingCleanup
+book init warpPendingSummary
 
 ### Database lists
@@ -19 +20 @@
 $warp_revert_overlap_DB = 0
 $warp_revert_warped_DB = 0
+$warpSummary_DB = 0
 
 ### Check status of warping tasks
@@ -30 +32 @@
   book init warpInputExp
   book init warpPendingSkyCell
+  book init warpPendingSummary
 end
 
@@ -55 +58 @@
     active true
   end
+  task warp.summary.load
+    active true
+  end
+  task warp.summary.run
+    active true
+  end
 end
 
@@ -78 +87 @@
   end
   task warp.revert.warped
+    active false
+  end
+  task warp.summary.load
+    active false
+  end
+  task warp.summary.run
     active false
   end
@@ -525 +540 @@
   end
 end
+
+### Load tasks for doing the warps
+### Tasks are loaded into warpPendingSkyCell.
+task           warp.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/warp.summary.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = warptool -tosummary
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warpSummary_DB
+      $run = $run -dbname $DB:$warpSummary_DB
+      $warpSummary_DB ++
+      if ($warpSummary_DB >= $DB:n) set warpSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    # XXX change tess_id to tess_dir when db is updated
+    ipptool2book stdout warpPendingSummary -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook warpPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup warpPendingSummary
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for calculating the warp overlaps
+### Tasks are taken from warpPendingSkyCell.
+task           warp.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages warpPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in warpPendingSkyCell (pantaskState == INIT)
+    book getpage warpPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword warpPendingSummary $pageName pantaskState RUN
+    book getword warpPendingSummary $pageName warp_id -var WARP_ID
+    book getword warpPendingSummary $pageName camera -var CAMERA
+    book getword warpPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword warpPendingSummary $pageName dbname -var DBNAME
+    book getword warpPendingSummary $pageName tess_id -var TESS_DIR
+    book getword warpPendingSummary $pageName exp_tag -var EXP_TAG
+    book getword warpPendingSummary $pageName state -var RUN_STATE
+
+    # set the host and workdir based on the skycell hash
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+#    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+#    if ("$PATH_BASE" == "NULL") 
+        ## generate outroot specific to this exposure
+    basename $TESS_DIR -var TESS_ID
+
+    sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $TESS_ID
+
+
+    stdout $LOGDIR/warp.summary.log
+    stderr $LOGDIR/warp.summary.log
+
+    $run = skycell_jpeg.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit warpPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword warpPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword warpPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end