Changeset 34839 for branches/eam_branches/ipp-20121130/ippTasks

Timestamp:

Dec 18, 2012, 5:57:19 AM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20121130/ippTasks

Files:

• . (modified) (1 prop)
• Makefile.am (modified) (1 diff)
• addstar.pro (modified) (2 diffs)
• bg.regeneration.pro (copied) (copied from trunk/ippTasks/bg.regeneration.pro )
• nightly_science.pro (modified) (1 diff)
• science.cleanup.pro (modified) (8 diffs)
• skycal.pro (modified) (7 diffs)
• staticsky.pro (modified) (2 diffs)
• summit.copy.pro (modified) (1 diff)
• survey.pro (modified) (6 diffs)

branches/eam_branches/ipp-20121130/ippTasks/Makefile.am

@@ -45 +45 @@
         diffphot.pro \
         lap.pro \
-        vp.pro
+        vp.pro \
+        bg.regeneration.pro
 
 other_files = \

branches/eam_branches/ipp-20121130/ippTasks/addstar.pro

@@ -212 +212 @@
   host         local
   periods      -poll $LOADPOLL
-  periods      -exec 300
+  periods      -exec $LOADEXEC_ADD
   periods      -timeout 300
   npending     1
@@ -266 +266 @@
   host         local
   periods      -poll $LOADPOLL
-  periods      -exec 300
+  periods      -exec $LOADEXEC_ADD
   periods      -timeout 300
   npending     1

branches/eam_branches/ipp-20121130/ippTasks/nightly_science.pro

@@ -258 +258 @@
 task              ns.dqstats.load
   host            local
-  periods         -poll 3600
+  periods         -poll $LOADPOLL
   periods         -exec $LOADEXEC
   periods         -timeout 300

branches/eam_branches/ipp-20121130/ippTasks/science.cleanup.pro

@@ -13 +13 @@
 
 check.globals
+
+# if set tell cleanup script to check for files from all components regardless of the data_state
+$CHECK_ALL_COMPONENTS = 0
+macro set.check.all.components
+    $CHECK_ALL_COMPONENTS = 1
+end
+macro clear.check.all.components
+    $CHECK_ALL_COMPONENTS = 0
+end
+macro get.check.all.components
+    echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS
+end
+
 
 ## chip.cleanup.pro : -*- sh -*-
@@ -130 +143 @@
     book getword chipPendingCleanup $pageName state -var CLEANUP_MODE
     book getword chipPendingCleanup $pageName chip_id -var CHIP_ID
+    book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword chipPendingCleanup $pageName dbname -var DBNAME
 
-    # specify choice of local or remote host based on camera and chip (class_id)
+    # specify choice of local or remote host based on camera and chip (in this case FPA)
     set.host.for.camera $CAMERA FPA
+
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
 
     stdout $LOGDIR/chip.cleanup.log
     stderr $LOGDIR/chip.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run
 
@@ -282 +303 @@
     book getword camPendingCleanup $pageName state  -var CLEANUP_MODE
     book getword camPendingCleanup $pageName cam_id -var CAM_ID
+    book getword camPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword camPendingCleanup $pageName dbname -var DBNAME
 
@@ -287 +310 @@
     set.host.for.camera $CAMERA FPA
 
+    # set the WORKDIR variable
+    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    ## generate LOGFILE specific to this exposure (& cam_id)
+    sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
+
     stdout $LOGDIR/camera.cleanup.log
     stderr $LOGDIR/camera.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     add_standard_args run
 
@@ -591 +619 @@
     book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
     book getword warpPendingCleanup $pageName warp_id -var WARP_ID
+    book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG
+    book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword warpPendingCleanup $pageName dbname -var DBNAME
 
     # specify choice of local or remote host based on camera and warp (class_id)
     set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID
 
     stdout $LOGDIR/warp.cleanup.log
     stderr $LOGDIR/warp.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run
 
@@ -748 +783 @@
     book getword diffCleanup $pageName state -var CLEANUP_MODE
     book getword diffCleanup $pageName diff_id -var DIFF_ID
+    book getword diffCleanup $pageName tess_id -var TESS_ID
+    book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE
     book getword diffCleanup $pageName dbname -var DBNAME
 
     # specify choice of local or remote host based on camera and diff (class_id)
     set.host.for.camera $CAMERA FPA
+    set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+    sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID
+
 
     stdout $LOGDIR/diff.cleanup.log
     stderr $LOGDIR/diff.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE
+    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
+    if ($CHECK_ALL_COMPONENTS)
+        $run = $run --check-all
+    end
     add_standard_args run
 
@@ -899 +941 @@
     book getword stackCleanup $pageName state -var CLEANUP_MODE
     book getword stackCleanup $pageName stack_id -var STACK_ID
+    book getword stackCleanup $pageName path_base -var PATH_BASE
     book getword stackCleanup $pageName dbname -var DBNAME
 
@@ -907 +950 @@
     stderr $LOGDIR/stack.cleanup.log
 
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE
+    sprintf LOGFILE "%s.cleanup.log" PATH_BASE
+    $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     add_standard_args run
 

branches/eam_branches/ipp-20121130/ippTasks/skycal.pro

@@ -8 +8 @@
 # test for required global variables
 check.globals
+
+# skcal.pro should have a more restricted polling limit (to avoid stress with getstar)
+if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 25
+
+macro set.skycal.poll
+  if ($0 != 2)
+    echo "USAGE:set.skycal.poll (value)"
+    break
+  end
+ 
+  $POLL_LIMIT_SKYCAL = $1
+end
+
+macro get.skycal.poll
+  echo "skycal poll limit: $POLL_LIMIT_SKYCAL"
+end
 
 ### Initialise the books containing the tasks to do
@@ -79 +95 @@
       # save the DB name for the exit tasks
       option $DB:$skycal_DB
-      $run = $run -dbname $DB:$skycal_DB -limit 40
+      $run = $run -dbname $DB:$skycal_DB
       $skycal_DB ++
       if ($skycal_DB >= $DB:n) set skycal_DB = 0
@@ -85 +101 @@
     add_poll_args run
     add_poll_labels run
+    $run = $run -limit $POLL_LIMIT_SKYCAL
     command $run
   end
@@ -123 +140 @@
 
   task.exec
+    # if we are unable to run use "long" exectime
+    periods -exec $RUNEXEC
     book npages skycalRun -var N
     if ($N == 0) break
     if ($NETWORK == 0) break
     if ($BURNTOOLING == 1) break
+
 
     # look for new entries in skycalRun (pantaskState == INIT)
@@ -147 +167 @@
 
     # set the host and workdir based on the skycell hash
-    set.host.for.skycell $SKYCELL_ID
+    # set.host.for.skycell $SKYCELL_ID
+    host anyhost
     set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
 
@@ -172 +193 @@
       echo command $run
     end
+    # since we have work to do shorten exec time
+    periods -exec .1
     command $run
   end
@@ -197 +220 @@
   host         local
 
-  periods      -poll 60.0
-  periods      -exec 1800.0
+  periods      -poll 10.0
+  periods      -exec 1200.0
   periods      -timeout 120.0
   npending     1
-  active false
+  active true
   
   stdout NULL

branches/eam_branches/ipp-20121130/ippTasks/staticsky.pro

@@ -70 +70 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -exec 30
+  periods      -timeout 60
   npending     1
 
@@ -154 +154 @@
 
     # set the host and workdir based on the skycell hash
-    set.host.for.skycell $SKYCELL_ID
+    # set.host.for.skycell $SKYCELL_ID
+    host anyhost
     set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
 

branches/eam_branches/ipp-20121130/ippTasks/summit.copy.pro

@@ -223 +223 @@
     periods      -exec     10
     periods      -poll     1
-    periods      -timeout  20
+    periods      -timeout  60
     npending     1
 

branches/eam_branches/ipp-20121130/ippTasks/survey.pro

@@ -17 +17 @@
  book create SURVEY_PUBLISH
  book create SURVEY_STATICSKYSINGLE 
+ book create SURVEY_SKYCAL
  $haveSurveyBooks = TRUE
 end
@@ -31 +32 @@
 $SURVEY_WARP_BG_DB = 0
 $SURVEY_PUBLISH_DB = 0
+$SURVEY_SKYCAL_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
 $SURVEY_EXEC = 120
@@ -73 +75 @@
     active true
   end  
+  task survey.skycal
+    active true
+  end
 end
 
@@ -110 +115 @@
   end
   task survey.staticskysingle
+    active false
+  end
+  task survey.skycal
     active false
   end
@@ -464 +472 @@
   book delpage SURVEY_STATICSKYSINGLE $1
 end
+
+macro survey.add.skycal
+  if ($0 != 3)
+    echo "USAGE: survey.add.skycal (label) (dist_group)"
+    break
+  end
+  book newpage SURVEY_SKYCAL $1
+  book setword SURVEY_SKYCAL $1 LABEL $1
+  book setword SURVEY_SKYCAL $1 DIST_GROUP $2
+  book setword SURVEY_SKYCAL $1 STATE PENDING
+end
+
+macro survey.del.skycal
+  if ($0 != 2)
+    echo "USAGE: survey.del.skycal (label)"
+    break
+  end
+  book delpage SURVEY_SKYCAL $1
+end
+
+macro survey.show.skycal
+  if ($0 != 1)
+    echo "USAGE: survey.show.skyacl"
+    break
+  end
+  book listbook SURVEY_SKYCAL
+end
+
 
 task survey.diff
@@ -1453 +1489 @@
 end
 
+# Survey task for Skycal stage
+
+task survey.skycal
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.skycal.log
+  stderr $LOGDIR/survey.skycal.log
+
+  task.exec
+    book npages SURVEY_SKYCAL -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_SKYCAL $i -var label
+        book setword SURVEY_SKYCAL $label STATE NEW
+      end
+      book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_SKYCAL_DB ++
+      if ($SURVEY_SKYCAL_DB >= $DB:n) set SURVEY_SKYCAL_DB = 0
+    end
+
+    book setword SURVEY_SKYCAL $label STATE DONE
+    book getword SURVEY_SKYCAL $label DIST_GROUP -var dist_group
+
+    # note: currently skycal uses the staticskyRun workdir 
+
+    $run = staticskytool -defineskycalrun -select_label $label -set_dist_group $dist_group
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_SKYCAL_DB
+      option $DB:$SURVEY_SKYCAL_DB
+    end
+    
+    # echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end