Changeset 30489 for trunk/ippTasks

Timestamp:

Feb 3, 2011, 2:51:02 PM (15 years ago)

Author:

bills

Message:

Changes to support storing distribution bundles in nebulous

Location:

trunk/ippTasks

Files:

• dist.pro (modified) (6 diffs)
• rcserver.pro (modified) (1 diff)
• survey.pro (modified) (3 diffs)

trunk/ippTasks/dist.pro

@@ -95 +95 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro get.host.for.component
     if ($0 != 3)
@@ -122 +123 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro set.dist.workdir.by.component
     if ($0 != 5)
@@ -139 +141 @@
     book getword ipphosts distribution count -var count
     if ("$count" == "NULL")
-        echo "ERROR: distributin hosts list is empty"
+        echo "ERROR: distribution hosts list is empty"
         break
     end
@@ -289 +291 @@
     end
 
-
-    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR 
-    if ("$OUTDIR" == "NULL")
-        echo ERROR failed to set workdir for $COMPONENT
-        break
-    end
-
     substr $COMPONENT 0 3 COMP_HEAD
     if ("$COMP_HEAD" == "sky")
         set.host.for.skycell $COMPONENT
+        set.workdir.by.skycell $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
     else 
-        # assume component is a class_id, if not we will default to anyhost
+        # assume component is a class_id, if  it is not we will default to
+        # host "anyhost"
+        # and volume "any"
         set.host.for.camera $CAMERA $COMPONENT
+        set.workdir.by.camera $CAMERA $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
+    end
+
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
     end
 
     sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
-#    stdout $logfile
-#    stderr $logfile
 
     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
@@ -401 +403 @@
     book getpage distToAdvance 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
+
+    # XXX the sql for disttool -toadvance does not include the camera:
+    # hack around this for now
+    $CAMERA = GPC1
 
     book setword distToAdvance $pageName pantaskState RUN
@@ -414 +420 @@
     end
 
-    host anyhost
-
-    # using $DIST_ID as the "component" works fine here since we only look
-    # at the last two digits. But make sure that there 2 digits
-    $fake_component = $STAGE_ID + 10
-    set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    substr $OUTDIR_TEMPLATE 0 3 SCHEME
+    if ("$SCHEME" == "neb")
+        # XXX This does not work if workdir is not in nebulous
+        # and $default_host == "any" because /data/any does not exist
+        # use "dummy" as component. Result will be 'any'
+        # set.workdir.by.camera $CAMERA dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+        set.workdir.by.camera GPC1 dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+    else 
+        # using $DIST_ID as the "component" works fine here since we only look
+        # at the last two digits. But make sure that there 2 digits
+        $fake_component = $STAGE_ID + 10
+        set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    end
+ 
     if ("$OUTDIR" == "NULL")
         echo ERROR failed to set workdir for $DIST_ID
         break
     end
+    host anyhost
 
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
-#    stdout $logfile
-#    stderr $logfile
-
-    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
+
+    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --camera $CAMERA --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
     add_standard_args run
 

trunk/ippTasks/rcserver.pro

@@ -136 +136 @@
 
     sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID
-    stdout $logfile
-    stderr $logfile
 
     book setword rcPendingFS $pageName pantaskState RUN
 
-    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
+    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR --logfile $logfile
 
     add_standard_args run

trunk/ippTasks/survey.pro

@@ -287 +287 @@
 # user functions to manipulate dist labels
 macro survey.add.dist
-  if ($0 != 2)
-    echo "USAGE: survey.add.dist (label)"
+  if ($0 != 3)
+    echo "USAGE: survey.add.dist (label) (workdir)"
     break
   end
   book newpage SURVEY_DIST $1
+  book setword SURVEY_DIST $1 WORKDIR $2
   book setword SURVEY_DIST $1 STATE PENDING
 end
@@ -896 +897 @@
     end
 
+    book getword SURVEY_DIST $label WORKDIR -var workdir_base
+    $year = `date +%Y`
+    $month = `date +%m`
+    $day = `date +%d`
+    $workdir = $workdir_base/$label/$year/$month/$day
+
     book setword SURVEY_DIST $label STATE DONE
   
     # note workdir is set by the script based on site.config
-    $run = dist_defineruns.pl --label $label
+    $run = dist_defineruns.pl --label $label --workdir $workdir
 
     if ($DB:n == 0)
@@ -908 +915 @@
     end
     
-#    echo $run
+    if ($VERBOSE > 1 )
+        echo $run
+    end
     command $run
   end