Changeset 42712 for trunk/ippTasks

Timestamp:

Jul 18, 2024, 2:55:58 PM (2 years ago)

Author:

cclin33

Message:

fix mis-match mosaic issue when running addstar, optimize addstar program

Location:

trunk/ippTasks

Files:

• addstar.pro (modified) (10 diffs)
• minidvodb.pro (modified) (8 diffs)

trunk/ippTasks/addstar.pro

@@ -228 +228 @@
 
 macro stuff
-   echo $LOADEXEC_ADD
-   echo $LOADPOLL
+   echo LOADEXEC_ADD: $LOADEXEC_ADD
+   echo LOADPOLL_ADD: $LOADPOLL_ADD
+   echo RUNEXEC_ADD:  $RUNEXEC_ADD
+   echo RUNPOLL_ADD:  $RUNPOLL_ADD
 end
 
@@ -249 +251 @@
 $addstar_revert_DB_DF = 0 
 
-#$LOADEXEC_ADD = 300
+# loading time every N seconds
 $LOADEXEC_ADD = 10
+$LOADPOLL_ADD = 10
+$RUNEXEC_ADD = 5
+$RUNPOLL_ADD = 10
 
 if ($?addstar_multiadd_limit == 0) set addstar_multiadd_limit = 0
@@ -273 +278 @@
   host         local
 
-  periods      -poll $LOADPOLL
+  periods      -poll $LOADPOLL_ADD
   periods      -exec $LOADEXEC_ADD
   periods      -timeout 300
   npending     1
 
+  date -var rundate
+
   stdout NULL
   stderr $LOGDIR/addstar.exp.log
 
   task.exec
-    #if no stages defined we can't run
+    #if no stages defined we cant run
     if ($ADDSTAGES:n == 0) echo "what" 
- #   if ($ADDSTAGES:n == 0) break
-    $run = addtool -pendingexp
+    if ($ADDSTAGES:n == 0) break
+    $run = addtool -pendingexp -addrand
+
+    book npages addPendingExp  -var addPendingExp_Npage
+    if ($addPendingExp_Npage > 10000)
+      $POLL_LIMIT = 1
+      echo $addPendingExp_Npage $rundate
+    end
     
     #option $ADDSTAGES:$addstar_stages_DB
@@ -301 +314 @@
     add_poll_args run
     add_poll_labels run
-    echo $run
+    echo $run $rundate
     command $run
   end
@@ -348 +361 @@
 
   task.exec
-    #if no stages defined we can't run
+    #if no stages defined we cant run
     if ($MULTIADDSTAGES:n == 0) echo "no stages for addstar.multi, use add.addstagesmulti" 
  #   if ($MULTIADDSTAGES:n == 0) break
@@ -400 +413 @@
 # run the addstar script on pending exposures
 task           addstar.exp.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
+  periods      -poll $RUNPOLL_ADD
+  periods      -exec $RUNEXEC_ADD
   periods      -timeout 1200
 
   ## we want only a single outstanding addstar job.  
 #  host         local
-  npending     100
+  npending     1000
 
   task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC_ADD
+
     book npages addPendingExp -var N
     if ($N == 0) break
@@ -433 +449 @@
     book getword addPendingExp $pageName image_only -var IMAGE_ONLY
     book getword addPendingExp $pageName dbname -var DBNAME
-    book getword addPendingExp $pageName dvodbhost  -var DVODBHOST
-
-    host -required $DVODBHOST
+    book getword addPendingExp $pageName addrun_host  -var ADDRUN_HOST
+
+    host -required $ADDRUN_HOST
 
     # specify choice of remote host based on camera and chip (class_id)
@@ -480 +496 @@
     stderr $LOGDIR/addstar.exp.log
 
-    $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --dvodbhost $DVODBHOST
+    $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --addrun_host $ADDRUN_HOST
     if ("$REDUCTION" != "NULL")
       $run = $run --reduction $REDUCTION
@@ -501 +517 @@
     end
     if ("$STAGE" == "fullforce_summary")
-    # This shouldn't need a stage_extra1, as there are no subcomponents to the ff summary.
+    # This shouldnt need a stage_extra1, as there are no subcomponents to the ff summary.
       $run = $run --stage_id $STAGE_ID
     end
-
 
 
@@ -528 +543 @@
       echo command $run
     end
+    periods -exec 0.1
     command $run
   end

trunk/ippTasks/minidvodb.pro

@@ -7 +7 @@
 
 
-#There is a book for each task, because I don't use labels. 
+#There is a book for each task, because I dont use labels. 
 
 if (not($?haveminidvodbBooks))
@@ -129 +129 @@
 end
 
-## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run.
+## you get no choice - you add all of them in at the same time. you can always turn off the tasks you dont want to run.
 macro add.minidvodb
   if ($0 != 8)
@@ -146 +146 @@
   book setword MINIDVODB_PREMERGE $1 DVODB $3
   book setword MINIDVODB_PREMERGE $1 CAMERA $6
+  book setword MINIDVODB_PREMERGE $1 MINIDVODB_HOST $7
   book setword MINIDVODB_PREMERGE $1 STATE PENDING
   #active  
@@ -153 +154 @@
   book setword MINIDVODB_ACTIVE $1 STATE PENDING
 
-  #create  note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out.
+  #create  note that camera should be GPC1 for it to work. I couldnt figure out how to easily get this out.
   book newpage MINIDVODB_CREATE $1
   book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1
@@ -187 +188 @@
   echo "minidvodb create"
   book listbook MINIDVODB_CREATE 
-  echo "minidvodb_active"
+  echo "minidvodb active"
   book listbook MINIDVODB_ACTIVE
 end
 
 
-#$LOADEXEC_MDB = 30
-$LOADEXEC_MDB = 2
-$LOADPOLL = 2
+$LOADEXEC_MDB = 60
 $timeout_mdb  = 2
-$LOADPOLL = 2
-$RUNEXEC = 2
+$LOADPOLL = 5
+$RUNEXEC = 30
 
 
@@ -430 +429 @@
   #we only want one running at a time
 
-  host         local
-  npending     1
+  npending     100
 
   task.exec
@@ -448 +446 @@
     book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
     book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH
+    book getword minidvodbPreMergelist $pageName minidvodb_host -var MINIDVODB_HOST
     book getword minidvodbPreMergelist $pageName camera -var CAMERA
     book getword minidvodbPreMergelist $pageName state -var STATE
@@ -455 +454 @@
     #still buggy
 
-    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
+    host -required $MINIDVODB_HOST
+
+    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID --minidvodb_host $MINIDVODB_HOST
     
   if ($DB:n == 0)