Changeset 30579 for branches/eam_branches/ipp-20101205/ippTasks

Timestamp:

Feb 11, 2011, 2:49:34 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20101205/ippTasks

Files:

• diskbalance.pro (modified) (1 diff)
• dist.pro (modified) (6 diffs)
• pstamp.pro (modified) (9 diffs)
• rcserver.pro (modified) (3 diffs)
• register.pro (modified) (5 diffs)
• survey.pro (modified) (4 diffs)

branches/eam_branches/ipp-20101205/ippTasks/diskbalance.pro

@@ -194 +194 @@
     book getword balancePending $pageName source_name      -var SOURCE
     book getword balancePending $pageName destination_name -var DESTINATION
-    
+    book getword balancePending $pageName destination_host -var DEST_HOST
     # Fix this with multihost restriction when possible.
-    set.host.for.balance $DESTINATION
+    set.host.for.balance $DEST_HOST
 
     stdout NULL

branches/eam_branches/ipp-20101205/ippTasks/dist.pro

@@ -95 +95 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro get.host.for.component
     if ($0 != 3)
@@ -122 +123 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro set.dist.workdir.by.component
     if ($0 != 5)
@@ -139 +141 @@
     book getword ipphosts distribution count -var count
     if ("$count" == "NULL")
-        echo "ERROR: distributin hosts list is empty"
+        echo "ERROR: distribution hosts list is empty"
         break
     end
@@ -289 +291 @@
     end
 
-
-    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR 
-    if ("$OUTDIR" == "NULL")
-        echo ERROR failed to set workdir for $COMPONENT
-        break
-    end
-
     substr $COMPONENT 0 3 COMP_HEAD
     if ("$COMP_HEAD" == "sky")
         set.host.for.skycell $COMPONENT
+        set.workdir.by.skycell $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
     else 
-        # assume component is a class_id, if not we will default to anyhost
+        # assume component is a class_id, if  it is not we will default to
+        # host "anyhost"
+        # and volume "any"
         set.host.for.camera $CAMERA $COMPONENT
+        set.workdir.by.camera $CAMERA $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
+    end
+
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
     end
 
     sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
-#    stdout $logfile
-#    stderr $logfile
 
     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
@@ -401 +403 @@
     book getpage distToAdvance 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
+
+    # XXX the sql for disttool -toadvance does not include the camera:
+    # hack around this for now
+    $CAMERA = GPC1
 
     book setword distToAdvance $pageName pantaskState RUN
@@ -414 +420 @@
     end
 
-    host anyhost
-
-    # using $DIST_ID as the "component" works fine here since we only look
-    # at the last two digits. But make sure that there 2 digits
-    $fake_component = $STAGE_ID + 10
-    set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    substr $OUTDIR_TEMPLATE 0 3 SCHEME
+    if ("$SCHEME" == "neb")
+        # XXX This does not work if workdir is not in nebulous
+        # and $default_host == "any" because /data/any does not exist
+        # use "dummy" as component. Result will be 'any'
+        # set.workdir.by.camera $CAMERA dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+        set.workdir.by.camera GPC1 dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+    else 
+        # using $DIST_ID as the "component" works fine here since we only look
+        # at the last two digits. But make sure that there 2 digits
+        $fake_component = $STAGE_ID + 10
+        set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    end
+ 
     if ("$OUTDIR" == "NULL")
         echo ERROR failed to set workdir for $DIST_ID
         break
     end
+    host anyhost
 
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
-#    stdout $logfile
-#    stderr $logfile
-
-    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
+
+    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --camera $CAMERA --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
     add_standard_args run
 

branches/eam_branches/ipp-20101205/ippTasks/pstamp.pro

@@ -18 +18 @@
 $pstampStopFaulted_DB = 0
 
+# give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT
+$PSTAMP_MAX_FAULT_COUNT = 5
+macro set.max.fault.count
+    $PSTAMP_MAX_FAULT_COUNT = $1
+end
+macro get.max.fault.count
+    echo maximum fault count: $PSTAMP_MAX_FAULT_COUNT
+end
+$POLL_DEP = 500
+
+macro set.dependent.poll
+    $POLL_DEP = $1
+end
+macro get.dependent.poll
+    echo dependent poll limit: $POLL_DEP
+end
+
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
 # warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word
@@ -137 +154 @@
     end
 end
+
 macro pstamp.find.on
     task pstamp.request.find
@@ -143 +161 @@
 end
 macro pstamp.find.off
-    task pstamp.request.find.load
-        active false
+    task pstamp.request.find
+        active false
+    end
+end
+
+macro pstamp.dependent.on
+    task pstamp.dependent.load
+        active true
+    end
+    task pstamp.dependent.run
+        active true
+    end
+end
+macro pstamp.dependent.off
+    task pstamp.dependent.load
+        active false
+    end
+    task pstamp.dependent.run
+        active false
+    end
+end
+
+
+macro pstamp.status.on
+    task pstamp.save.status
+        active true
+    end
+end
+macro pstamp.status.off
+    task pstamp.save.status
+        active false
+    end
+end
+
+macro pstamp.status.set.exec
+    task pstamp.save.status
+        periods -exec $1
     end
 end
@@ -509 +562 @@
             echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS
         end
-        showcommand failure
+#        showcommand failure
         process_exit pstampJob $options:0 $JOB_STATUS
     end
@@ -593 +646 @@
         add_poll_args run
         add_poll_labels run 
-        command $run
+        command $run -limit $POLL_DEP
     end
 
@@ -648 +701 @@
         book getword pstampDependent $pageName outdir     -var OUTDIR
         book getword pstampDependent $pageName need_magic -var NEED_MAGIC
+        book getword pstampDependent $pageName fault_count -var FAULT_COUNT
         book getword pstampDependent $pageName dbname     -var DBNAME
 
@@ -666 +720 @@
         stderr $MYLOGFILE
 
-        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC
+        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT
 
         add_standard_args run
@@ -689 +743 @@
             echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS
         end
-        showcommand failure
+#        showcommand failure
         process_exit pstampDependent $options:0 $JOB_STATUS
     end
@@ -905 +959 @@
     end
 end
+
+task pstamp.save.status
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec 90
+    periods     -timeout 20
+    npending    1
+
+    task.exec
+        stdout NULL
+        stderr $LOGSUBDIR/pstamp.save.status.log
+
+        $run = pstamp_save_server_status.pl --update-link
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        # echo nothing to do
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end

branches/eam_branches/ipp-20101205/ippTasks/rcserver.pro

@@ -40 +40 @@
     active false
   end
-  task rcserver.revert
-    active false
-  end
 end
 
@@ -48 +45 @@
   task rcserver.revert
     active true
+  end
+end
+
+macro rcserver.revert.off
+  task rcserver.revert
+    active false
   end
 end
@@ -136 +139 @@
 
     sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID
-    stdout $logfile
-    stderr $logfile
 
     book setword rcPendingFS $pageName pantaskState RUN
 
-    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
+    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR --logfile $logfile
 
     add_standard_args run

branches/eam_branches/ipp-20101205/ippTasks/register.pro

@@ -105 +105 @@
 
 macro register.add.date
-  if ($0 != 2) 
-    echo "USAGE: register.add.date (YYYY-MM-DD)"
+  if ($0 != 4) 
+    echo "USAGE: register.add.date (YYYY-MM-DD) (DBNAME) (VALID_BURNTOOL)"
     break
   end
@@ -112 +112 @@
    book newpage regDates $1
    book setword regDates $1 nsState NEW
-   book setword regDates $1 dbname $DB:0 
-end
-
-
-$valid_burntool_value = 14
+   book setword regDates $1 dbname $2
+   book setword regDates $1 valid_burntool $3
+end
+
+
+# $valid_burntool_value = 14
 $sunrise = 17:30:00
 $sunset  = 03:30:00
@@ -565 +566 @@
      if ($Npages == 0) 
        $today = `date -u +%Y-%m-%d`
+       $dbname = $DB:0
+       $valid_burntool = 14
      else 
        book getpage regDates $reg_datePAGE -var today
+       book getword regDates $today dbname -var dbname
+       book getword regDates $today valid_burntool -var valid_burntool
        $reg_datePAGE ++
        if ($reg_datePAGE >= $Npages) set reg_datePAGE = 0
@@ -577 +582 @@
     $dateobs_begin = $today\T$sunset
     $dateobs_end   = $today\T$sunrise
-    $run = $run -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end -valid_burntool $valid_burntool_value
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$regPendingBurntoolImfile_DB
-      $run = $run -dbname $DB:$regPendingBurntoolImfile_DB
-      $regPendingBurntoolImfile_DB ++
-      if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0
-    end
+    $run = $run -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end -valid_burntool $valid_burntool
+    $run = $run -dbname $dbname
+    option $dbname
+#     if ($DB:n == 0)
+#       option DEFAULT
+#     else
+#       # save the DB name for the exit tasks
+#       option $DB:$regPendingBurntoolImfile_DB
+#       $run = $run -dbname $DB:$regPendingBurntoolImfile_DB
+#       if ("$dbname" != "$DB:$regPendingBurntoolImfile_DB")
+#         break
+#       end
+#       $regPendingBurntoolImfile_DB ++
+#       if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0
+#     end
 
 #    echo $run
@@ -697 +707 @@
     book setword regDates $today nsState NEW
     book setword regDates $today dbname $DB:0
+    book setword regDates $today valid_burntool 14
 
     command true

branches/eam_branches/ipp-20101205/ippTasks/survey.pro

@@ -287 +287 @@
 # user functions to manipulate dist labels
 macro survey.add.dist
-  if ($0 != 2)
-    echo "USAGE: survey.add.dist (label)"
+  if ($0 != 3)
+    echo "USAGE: survey.add.dist (label) (workdir)"
     break
   end
   book newpage SURVEY_DIST $1
+  book setword SURVEY_DIST $1 WORKDIR $2
   book setword SURVEY_DIST $1 STATE PENDING
 end
@@ -896 +897 @@
     end
 
+    book getword SURVEY_DIST $label WORKDIR -var workdir_base
+    $year = `date +%Y`
+    $month = `date +%m`
+    $day = `date +%d`
+    $workdir = $workdir_base/$label/$year/$month/$day
+
     book setword SURVEY_DIST $label STATE DONE
   
     # note workdir is set by the script based on site.config
-    $run = dist_defineruns.pl --label $label
+    $run = dist_defineruns.pl --label $label --workdir $workdir
 
     if ($DB:n == 0)
@@ -908 +915 @@
     end
     
-#    echo $run
+    if ($VERBOSE > 1 )
+        echo $run
+    end
     command $run
   end
@@ -1126 +1135 @@
     end
     
-    echo $run
+    # echo $run
 
     command $run