Changeset 33415 for branches/meh_branches/ppstack_test/ippTasks/survey.pro

Timestamp:

Mar 5, 2012, 5:19:48 PM (14 years ago)

Author:

mhuber

Message:

merging latest r33407 trunk changes to meh_branches/ppstack_test

Location:

branches/meh_branches/ppstack_test

Files:

• . (modified) (1 prop)
• ippTasks/survey.pro (modified) (22 diffs)

branches/meh_branches/ppstack_test/ippTasks/survey.pro

@@ -3 +3 @@
 # test for required global variables
 check.globals
-
-# there is a bug in 'getbook', use a different method below
-# book getbook SURVEY_DIFF -var isBook
-# if ("$isBook" != "SURVEY_DIFF")
-#   book create SURVEY_DIFF
-# end
-# book getbook SURVEY_MAGIC -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_MAGIC
-# end
-# book getbook SURVEY_DESTREAK -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_DESTREAK
-# end
-# book getbook SURVEY_DIST -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_DIST
-# end
 
 if (not($?haveSurveyBooks))
@@ -30 +12 @@
  book create SURVEY_DIST
  book create SURVEY_ADDSTAR   
+ book create SURVEY_MERGEDVODB
  book create SURVEY_CHIP_BG
  book create SURVEY_WARP_BG
  book create SURVEY_PUBLISH
- book create SURVEY_STATICSKYSINGLE   
+ book create SURVEY_STATICSKYSINGLE 
  $haveSurveyBooks = TRUE
 end
@@ -44 +27 @@
 $SURVEY_DIST_DB = 0
 $SURVEY_ADDSTAR_DB = 0
+$SURVEY_MERGEDVODB_DB = 0
 $SURVEY_CHIP_BG_DB = 0
 $SURVEY_WARP_BG_DB = 0
 $SURVEY_PUBLISH_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
-
 $SURVEY_EXEC = 120
 $SURVEY_POLL = 10
@@ -69 +52 @@
     active true
   end
+  task survey.mergedvodb
+    active true
+  end
   task survey.destreak
     active true
@@ -105 +91 @@
     active false
   end
+  task survey.mergedvodb
+    active false
+  end  
   task survey.destreak
     active false
@@ -213 +202 @@
 # user functions to manipulate magic labels
 macro survey.add.magic
-  if ($0 != 3)
-    echo "USAGE: survey.add.magic (label) (workdir base)"
+  if (($0 != 3)&&($0 != 4))
+    echo "USAGE: survey.add.magic (label) (workdir base) (multidiff)"
     break
   end
   book newpage SURVEY_MAGIC $1
   book setword SURVEY_MAGIC $1 WORKDIR $2
+  if ($0 == 3)
+     book setword SURVEY_MAGIC $1 MULTIDIFF FALSE
+  else 
+     book setword SURVEY_MAGIC $1 MULTIDIFF $3
+  end
   book setword SURVEY_MAGIC $1 STATE PENDING
 end
@@ -267 +261 @@
 end
 
+macro survey.add.mergedvodb
+  if ($0 != 4)
+    echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)"
+    break
+  end
+  book newpage SURVEY_MERGEDVODB $1
+  book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2
+  book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3
+end
+
+macro survey.del.mergedvodb
+  if ($0 != 2)
+    echo "USAGE: survey.del.mergedvodb (tag)"
+    break
+  end
+  book delpage SURVEY_MERGEDVODB $1
+end
+
+macro survey.show.mergedvodb
+  if ($0 != 1)
+    echo "USAGE: survey.show.mergedvodb"
+    break
+  end
+  book listbook SURVEY_MERGEDVODB
+end
+
+
 # user functions to manipulate destreak labels
 macro survey.add.destreak
@@ -297 +318 @@
 # user functions to manipulate dist labels
 macro survey.add.dist
-  if ($0 != 3)
-    echo "USAGE: survey.add.dist (label) (workdir)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.dist (label) (workdir) (muggle)"
     break
   end
   book newpage SURVEY_DIST $1
   book setword SURVEY_DIST $1 WORKDIR $2
+  book setword SURVEY_DIST $1 MUGGLE $3
   book setword SURVEY_DIST $1 STATE PENDING
 end
@@ -324 +346 @@
 # user functions to manipulate chip_bg labels
 macro survey.add.chip.bg
-  if ($0 != 3)
-    echo "USAGE: survey.add.chip.bg (label) (dist_group)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.chip.bg (label) (dist_group) (muggle)"
     break
   end
   book newpage SURVEY_CHIP_BG $1
   book setword SURVEY_CHIP_BG $1 DIST_GROUP $2
+  book setword SURVEY_CHIP_BG $1 MUGGLE $3
   book setword SURVEY_CHIP_BG $1 STATE PENDING
 end
@@ -351 +374 @@
 # user functions to manipulate warp_bg labels
 macro survey.add.warp.bg
-  if ($0 != 3)
-    echo "USAGE: survey.add.warp.bg (label) (dist_group)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.warp.bg (label) (dist_group) (muggle)"
     break
   end
   book newpage SURVEY_WARP_BG $1
   book setword SURVEY_WARP_BG $1 DIST_GROUP $2
+  book setword SURVEY_WARP_BG $1 MUGGLE $3
   book setword SURVEY_WARP_BG $1 STATE PENDING
 end
@@ -610 +634 @@
   periods      -exec $SURVEY_EXEC
   periods      -timeout $SURVEY_TIMEOUT
-  trange        13:00:00 14:00:00
+# 13:00 was being done in early morning rather than afternoon HST (so UT value), want both times to have SSdiff ASAP?
+# changing earlier for QUB to fetch @10pm for nightly processing there, should be enough time to finish MD stacks
+#  trange        13:00:00 14:00:00
+  trange        20:00:00 21:00:00
   npending      1
 
@@ -725 +752 @@
     book setword SURVEY_MAGIC $label STATE DONE
     book getword SURVEY_MAGIC $label WORKDIR -var workdir
-  
+    book getword SURVEY_MAGIC $label MULTIDIFF -var multidiff
+
     $run = magictool -rerun -definebyquery -label $label -diff_label $label -workdir $workdir/$label
+    
+    if (("$multidiff" == "TRUE")||("$multidiff" == "MULTIDIFF"))
+       $run = $run -multidiff
+    end
 
     if ($DB:n == 0)
@@ -735 +767 @@
     end
     
-    # echo $run
+    echo $run
     command $run
   end
@@ -814 +846 @@
     if ("$stage" == "cam") 
         #only queue destreaked cams. stacks and staticsky don't need this
-        if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
+    #    if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
             #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked
+           # now with no magic, all are uncensored
            $run = $run -uncensored
-        else
+#       else
            #if not isp, then run destreaked version only
-           $run = $run -destreaked
-        end
+#          $run = $run -destreaked
+#       end
     end
     if ("$stage" == "staticsky") 
@@ -826 +859 @@
         $run = $run -uncensored
     end
+    if ("$stage" == "staticsky_multi") 
+        #only queue uncensored staticsky (multi filter)
+        $run = $run -uncensored
+    end
+
     if ("$stage" == "stack") 
         #only queue uncensored stacks
@@ -858 +896 @@
   end
 end
+
+
+task survey.mergedvodb
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.mergedvodb.log
+  stderr $LOGDIR/survey.mergedvodb.log
+
+  task.exec
+    book npages SURVEY_MERGEDVODB -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
+    if ("$tag" == "NULL")
+      # All labels have been done --- reset
+    # echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_MERGEDVODB $i -var tag
+        book setword SURVEY_MERGEDVODB $tag STATE NEW
+      end
+      book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
+
+      # Select different database
+      $SURVEY_MERGEDVODB_DB ++
+      if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0
+    end
+   book setword SURVEY_MERGEDVODB $tag STATE DONE
+   book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb
+   book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group
+     
+ #   $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb
+    #$run = mergetool 
+    $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB
+      option $DB:$SURVEY_MERGEDVODB_DB
+    end
+    #this run doesn't work because there's no workdir 
+    #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group 
+    
+    
+    # echo $run
+
+  echo $run
+    command $run
+  end
+
+  # success
+   
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
 
 
@@ -974 +1090 @@
     $workdir = $workdir_base/$label/$year/$month/$day
 
+    book getword SURVEY_DIST $label MUGGLE -var muggle
+
     book setword SURVEY_DIST $label STATE DONE
   
     # note workdir is set by the script based on site.config
     $run = dist_defineruns.pl --label $label --workdir $workdir
+    if ($muggle != 0)
+        $run = $run --no_magic 
+    end
 
     if ($DB:n == 0)
@@ -1048 +1169 @@
     book setword SURVEY_CHIP_BG $label STATE DONE
     book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group
+    book getword SURVEY_CHIP_BG $label MUGGLE -var muggle
   
     $run = bgtool -definechip -label $label -set_dist_group $dist_group
+    if ($muggle == 0)
+        $run = $run -destreaked
+    end
 
     if ($DB:n == 0)
@@ -1058 +1183 @@
     end
     
-    # echo $run
+    if ($VERBOSE > 1) 
+        echo $run
+    end
     command $run
   end
@@ -1118 +1245 @@
     book setword SURVEY_WARP_BG $label STATE DONE
     book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group
+    book getword SURVEY_WARP_BG $label MUGGLE -var muggle
   
     $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group
+    if ($muggle == 0)
+        $run = $run -destreaked
+    end
+
 
     if ($DB:n == 0)
@@ -1128 +1260 @@
     end
     
-    # echo $run
+    if ($VERBOSE > 1) 
+        echo $run
+    end
     command $run
   end
@@ -1248 +1382 @@
     book npages SURVEY_STATICSKYSINGLE -var N
     if ($N == 0)
-      echo "No STATICSKYSINGLE labels for processing"
+       # echo "No STATICSKYSINGLE labels for processing"
        break
     endif