Changeset 32630 for branches/eam_branches/ipp-20110906/ippTasks

Timestamp:

Nov 8, 2011, 2:44:12 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20110906

Files:

• . (modified) (1 prop)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/destreak.pro (modified) (2 diffs)
• ippTasks/diskbalance.pro (modified) (5 diffs)
• ippTasks/ipphosts.mhpcc.config (modified) (7 diffs)
• ippTasks/lap.pro (modified) (4 diffs)
• ippTasks/nightly_science.pro (modified) (2 diffs)
• ippTasks/summit.copy.pro (modified) (1 diff)
• ippTasks/vp.pro (copied) (copied from trunk/ippTasks/vp.pro )

branches/eam_branches/ipp-20110906/ippTasks/Makefile.am

@@ -42 +42 @@
         background.pro \
         diffphot.pro \
-        lap.pro
+        lap.pro \
+        vp.pro
 
 other_files = \

branches/eam_branches/ipp-20110906/ippTasks/destreak.pro

@@ -206 +206 @@
     book getword magicToDS $pageName magicked -var MAGICKED
     book getword magicToDS $pageName dbname -var DBNAME
-
+    book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID
+    book getword magicToDS $pageName mismatched_tess -var MISMATCHED_TESS
 
     substr $COMPONENT 0 3 COMP_HEAD
@@ -224 +225 @@
     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE
 
+    if ($MISMATCHED_TESS) 
+        book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID
+        $run = $run --mismatched_tess --diff_tess_id $DIFF_TESS_ID
+    end
     add_standard_args run
 

branches/eam_branches/ipp-20110906/ippTasks/diskbalance.pro

@@ -15 +15 @@
 book init balanceControl
 
+
+book init targetPending
+book init targetControl
+
 macro balance.reset
   book init balancePending
@@ -41 +45 @@
 end
 
+macro target.reset
+  book init targetPending
+end
+
+macro target.status
+  book listbook targetPending
+end
+
+macro target.on
+  task target.load
+    active true
+  end
+  task target.run
+    active true
+  end
+end
+
+macro target.off
+  task target.load
+    active false
+  end
+  task target.run
+    active false
+  end
+end
+ 
 macro set.host.for.balance
   if ($0 != 2)
@@ -71 +101 @@
   book setword balanceControl control mDestinations $2
 end
+
+macro set.balance.range
+  if ($0 != 3)
+    echo "USAGE: set.balance.range (SO_ID_START) (SO_ID_RANGE)"
+    break
+  end
+
+  $BALANCE_SO_ID_START = $1
+  $BALANCE_SO_ID_RANGE = $2
+end
+
 
 macro show.balance.numbers
@@ -90 +131 @@
  end
  $BALANCE_TARGET = $1
+end
+
+macro set.target.range
+  if ($0 != 3)
+    echo "USAGE: set.target.range (SO_ID_START) (SO_ID_RANGE)"
+    break
+  end
+
+  $TARGET_SO_ID_START = $1
+  $TARGET_SO_ID_RANGE = $2
+end
+
+
+macro show.target.numbers
+  echo "target so_ids: $TARGET_SO_ID_START $TARGET_SO_ID_RANGE"
+end
+
+macro show.targetPending
+  book npages targetPending -var N
+  echo "$N pages in book"
+  book listbook targetPending
 end
 
@@ -250 +312 @@
 
 
-
-    
-
+# This value will need to be incremented periodically.  It looks like we grow by about 5e7 so_ids per month.
+$TARGET_BIG_SO_ID   = 6e8
+$TARGET_SO_ID_START = int($TARGET_BIG_SO_ID * rnd(0)) 
+$TARGET_SO_ID_RANGE = 500000
+$TARGET_LIMIT       = 500
+$TARGET_OFFSET      = 0
+$TARGET_TARGET      = "SOURCE"
+# Select Nebulous objects which should be shuffled
+task           target.load
+  host         local
+
+  periods      -poll 60
+  periods      -exec 60
+  periods      -timeout 1500
+  npending     1
+
+  # logs
+  stdout NULL
+  stderr $LOGSUBDIR/target.log
+    
+  task.exec
+    book npages targetPending -var N
+    if ($N > 2000)
+      process_cleanup targetPending
+      break
+    end
+
+    command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingtarget --limit $TARGET_LIMIT --offset $TARGET_OFFSET --so_id_start $TARGET_SO_ID_START --so_id_range $TARGET_SO_ID_RANGE  
+  end
+
+   # success : 0 -- we did not hit the limit, advance so_id counter
+  task.exit 0
+    # advance the so_id counter
+    $TARGET_SO_ID_START = $TARGET_SO_ID_START + $TARGET_SO_ID_RANGE
+    $TARGET_OFFSET = 0
+    # convert 'stdout' to book format
+    ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT
+
+    if ($VERBOSE > 2)
+      book listbook targetPending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup targetPending
+  end
+
+  # success : 1 -- we DID hit the limit, do NOT advance so_id counter 
+  task.exit 1
+    # convert 'stdout' to book format
+    ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT
+    $TARGET_OFFSET = $TARGET_OFFSET + $TARGET_LIMIT
+    if ($VERBOSE > 2)
+      book listbook targetPending
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup targetPending
+  end
+
+  # out of so_id range, reset
+  task.exit 10
+    # advance the so_id counter
+    $TARGET_SO_ID_START = 0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# task to execute shuffles
+task            target.run
+  periods       -poll 0.5
+  periods       -exec 5
+  periods       -timeout 5
+  npending      200 
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec 5
+
+    book npages targetPending -var N
+    if ($NETWORK == 0) break
+    if ($N == 0) break
+
+    # look for new objects in targetPending
+    book getpage targetPending 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword targetPending $pageName pantaskState RUN
+
+    book getword targetPending $pageName key              -var KEY
+    book getword targetPending $pageName source_name      -var SOURCE
+    book getword targetPending $pageName source_host      -var SOURCE_HOST
+    book getword targetPending $pageName destination_name -var DESTINATION
+    book getword targetPending $pageName destination_host -var DEST_HOST
+    book getword targetPending $pageName destination_uri  -var DEST_URI
+    book getword targetPending $pageName ins_id           -var INS_ID
+
+    # Fix this with multihost restriction when possible.
+    if ("$TARGET_TARGET" == "DESTINATION")
+       set.host.for.target $DEST_HOST
+    else 
+       set.host.for.target $SOURCE_HOST
+    end
+    
+
+    stdout NULL
+    stderr $LOGSUBDIR/target.log
+
+    $run = neb-insedit --key $KEY --ins_id $INS_ID --uri $DEST_URI --volume $DESTINATION 
+    $run = $run --db $NEB_DB --host $NEB_HOST --user $NEB_USER --pass $NEB_PASS
+    $run = $run --neb_host http://ippc04/nebulous
+    
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+        echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit     default
+    process_exit targetPending $options:0 $JOB_STATUS
+  end
+
+  task.exit    crash
+    showcommand crash
+    book setword targetPending $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword targetPending $options:0 pantaskState TIMEOUT
+  end
+end
+
+

branches/eam_branches/ipp-20110906/ippTasks/ipphosts.mhpcc.config

@@ -95 +95 @@
   camera STR GPC1
 
-  XY01  STR  ipp005
-  XY02  STR  ipp005
+  XY01  STR  ipp010
+  XY02  STR  ipp011
   XY03  STR  ipp006
   XY04  STR  ipp006
@@ -128 +128 @@
   XY33  STR  ipp021
   XY34  STR  ipp023
-  XY35  STR  ipp023
+  XY35  STR  ipp013
 
   XY36  STR  ipp024
-  XY37  STR  ipp024
+  XY37  STR  ipp019
   XY40  STR  ipp025
-  XY41  STR  ipp025
-
-  XY42  STR  ipp026
-  XY43  STR  ipp026
+  XY41  STR  ipp018
+
+  XY42  STR  ipp017
+  XY43  STR  ipp015
   XY44  STR  ipp027
   XY45  STR  ipp028
@@ -174 +174 @@
   camera STR gpc1
 
-  ota01  STR  ipp005
-  ota02  STR  ipp005
+  ota01  STR  ipp010
+  ota02  STR  ipp011
   ota03  STR  ipp006
   ota04  STR  ipp006
@@ -207 +207 @@
   ota33  STR  ipp021
   ota34  STR  ipp023
-  ota35  STR  ipp023
+  ota35  STR  ipp013
                     
   ota36  STR  ipp024
-  ota37  STR  ipp024
+  ota37  STR  ipp019
   ota40  STR  ipp025
-  ota41  STR  ipp025
-                    
-  ota42  STR  ipp026
-  ota43  STR  ipp026
+  ota41  STR  ipp018
+                    
+  ota42  STR  ipp017
+  ota43  STR  ipp015
   ota44  STR  ipp027
   ota45  STR  ipp028
@@ -285 +285 @@
   count S32   60
 
-  XY01  STR  ipp005
-  XY02  STR  ipp005
+  XY01  STR  ipp010
+  XY02  STR  ipp011
   XY03  STR  ipp006
   XY04  STR  ipp006
@@ -318 +318 @@
   XY33  STR  ipp021
   XY34  STR  ipp023
-  XY35  STR  ipp023
+  XY35  STR  ipp013
                    
   XY36  STR  ipp024
-  XY37  STR  ipp024
+  XY37  STR  ipp019
   XY40  STR  ipp025
-  XY41  STR  ipp025
-                   
-  XY42  STR  ipp026
-  XY43  STR  ipp026
+  XY41  STR  ipp018
+                   
+  XY42  STR  ipp017
+  XY43  STR  ipp015
   XY44  STR  ipp027
   XY45  STR  ipp028
@@ -441 +441 @@
   sky75 STR  ipp053
 END
+
+ipphosts METADATA
+  camera STR STARE
+
+  ota01  STR  ipp010
+  ota02  STR  ipp010
+  ota03  STR  ipp010
+  ota04  STR  ipp010
+                    
+  ota05  STR  ipp011
+  ota06  STR  ipp011
+  ota10  STR  ipp011
+  ota11  STR  ipp011
+                    
+  ota12  STR  ipp012
+  ota13  STR  ipp012
+  ota14  STR  ipp012
+  ota15  STR  ipp012
+                    
+  ota16  STR  ipp013
+  ota17  STR  ipp013
+  ota20  STR  ipp013
+  ota21  STR  ipp013
+                    
+  ota22  STR  ipp014
+  ota23  STR  ipp014
+  ota24  STR  ipp014
+  ota25  STR  ipp014
+                    
+  ota26  STR  ipp015
+  ota27  STR  ipp015
+  ota30  STR  ipp015
+  ota31  STR  ipp015
+                    
+  ota32  STR  ipp016
+  ota33  STR  ipp016
+  ota34  STR  ipp016
+  ota35  STR  ipp016
+                    
+  ota36  STR  ipp017
+  ota37  STR  ipp017
+  ota40  STR  ipp017
+  ota41  STR  ipp017
+                    
+  ota42  STR  ipp018
+  ota43  STR  ipp018
+  ota44  STR  ipp018
+  ota45  STR  ipp018
+                    
+  ota46  STR  ipp019
+  ota47  STR  ipp019
+  ota50  STR  ipp019
+  ota51  STR  ipp019
+                    
+  ota52  STR  ipp020
+  ota53  STR  ipp020
+  ota54  STR  ipp020
+  ota55  STR  ipp020
+                    
+  ota56  STR  ipp021
+  ota57  STR  ipp021
+  ota60  STR  ipp021
+  ota61  STR  ipp021
+                    
+  ota62  STR  ipp010
+  ota63  STR  ipp011
+  ota64  STR  ipp012
+  ota65  STR  ipp013
+                    
+  ota66  STR  ipp014
+  ota67  STR  ipp015
+  ota71  STR  ipp016
+  ota72  STR  ipp017
+                    
+  ota73  STR  ipp018
+  ota74  STR  ipp019
+  ota75  STR  ipp020
+  ota76  STR  ipp021
+END

branches/eam_branches/ipp-20110906/ippTasks/lap.pro

@@ -10 +10 @@
 $lap_RunPage = 0
 $lap_DonePage = 0
+
+$LAP_QUEUE = NULL
 
 book init lapNewRuns
@@ -135 +137 @@
  
 
+macro lap.define.queue
+  $LAP_QUEUE = $1
+end
+
 task           lap.initial.load
   host         local
@@ -388 +394 @@
     stderr $LOGDIR/lap.load.log
 
-    $run = laptool -pendingrun -state done
+    $run = laptool -pendingrun -state full
 
     if ($lapSeq:n != 0)
@@ -463 +469 @@
     option $LAP_ID
     $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID
+    if ("$LAP_QUEUE" != "NULL") 
+      $run = $run --queue_list $LAP_QUEUE
+    end
+    
 
     command $run

branches/eam_branches/ipp-20110906/ippTasks/nightly_science.pro

@@ -178 +178 @@
   periods         -poll $LOADPOLL
   periods         -exec $LOADEXEC
-  periods         -timeout 30
+  periods         -timeout 300
   trange          0:00:00 1:00:00 -nmax 1
   npending        1
@@ -260 +260 @@
   periods         -poll 3600
   periods         -exec $LOADEXEC
-  periods         -timeout 30
+  periods         -timeout 300
   trange          22:00:00 23:59:59
   trange          00:00:00 10:00:00

branches/eam_branches/ipp-20110906/ippTasks/summit.copy.pro

@@ -442 +442 @@
         # this function uses workdir_template, default_host, volume_template, volume_default,
         # it sets workdir and volume
-        set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base
+
+        # Look to see if we have a stare type observation, and redirect it to an appropriate host
+        substr $EXP_NAME 10 1 EXPTYPE_KEY
+        if ("$EXPTYPE_KEY" == "a") 
+          set.workdir.by.camera STARE $CLASS_ID $workdir_template $default_host workdir_base
+        else 
+          set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base
+        end
 
         # figure out filename