Changeset 32284

Timestamp:

Sep 1, 2011, 2:13:19 PM (15 years ago)

Author:

watersc1

Message:

Fix magictool to understand that we sometimes want to allow multiple diffs in the same label to be independently processed.

Location:

tags/ipp-20110622

Files:

• ippTasks/survey.pro (modified) (15 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/magictool_definebyquery_select.sql (modified) (1 diff)
• ippTools/share/magictool_definebyquery_select_multidiff.sql (copied) (copied from trunk/ippTools/share/magictool_definebyquery_select_multidiff.sql )
• ippTools/src/magictool.c (modified) (2 diffs)
• ippTools/src/magictoolConfig.c (modified) (1 diff)

tags/ipp-20110622/ippTasks/survey.pro

@@ -33 +33 @@
  book create SURVEY_WARP_BG
  book create SURVEY_PUBLISH
+ book create SURVEY_STATICSKYSINGLE   
  $haveSurveyBooks = TRUE
 end
@@ -46 +47 @@
 $SURVEY_WARP_BG_DB = 0
 $SURVEY_PUBLISH_DB = 0
+$SURVEY_STATICSKYSINGLE_DB = 0
 
 $SURVEY_EXEC = 120
@@ -82 +84 @@
     active true
   end
+  task survey.staticskysingle
+    active true
+  end  
 end
 
@@ -113 +118 @@
   end
   task survey.publish
+    active false
+  end
+  task survey.staticskysingle
     active false
   end
@@ -205 +213 @@
 # user functions to manipulate magic labels
 macro survey.add.magic
-  if ($0 != 3)
-    echo "USAGE: survey.add.magic (label) (workdir base)"
+  if (($0 != 3)&&($0 != 4))
+    echo "USAGE: survey.add.magic (label) (workdir base) (multidiff)"
     break
   end
   book newpage SURVEY_MAGIC $1
   book setword SURVEY_MAGIC $1 WORKDIR $2
+  if ($0 == 3)
+     book setword SURVEY_MAGIC $1 MULTIDIFF FALSE
+  else 
+     book setword SURVEY_MAGIC $1 MULTIDIFF $3
+  end
   book setword SURVEY_MAGIC $1 STATE PENDING
 end
@@ -231 +244 @@
 
 macro survey.add.addstar
-  if ($0 != 5)
-    echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group) (stage)"
+  if ($0 != 6)
+    echo "USAGE: survey.add.addstar (tag) (label) (dvodb) (minidvodb_group) (stage)"
     break
   end
   book newpage SURVEY_ADDSTAR $1
-  book setword SURVEY_ADDSTAR $1 DVODB $2
-  book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $3
-  book setword SURVEY_ADDSTAR $1 STAGE $4  
+  book setword SURVEY_ADDSTAR $1 LABEL $2
+  book setword SURVEY_ADDSTAR $1 DVODB $3
+  book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $4
+  book setword SURVEY_ADDSTAR $1 STAGE $5  
   book setword SURVEY_ADDSTAR $1 STATE PENDING
 end
@@ -244 +258 @@
 macro survey.del.addstar
   if ($0 != 2)
-    echo "USAGE: survey.del.addstar (label)"
+    echo "USAGE: survey.del.addstar (tag)"
     break
   end
@@ -396 +410 @@
 end
 
+macro survey.add.staticskysingle
+# adds each of the filters for the label chosen
+# for LAP - the final stacks chosen have %final% in them, and we 
+# want some ability to choose that
+# for others can use %
+
+ if ($0 != 7)
+    echo "USAGE: survey.add.staticskysingle (tag) (label) (workdir) (distgroup)  (selectdatagroup) (filter)"
+    break
+ end
+    book newpage SURVEY_STATICSKYSINGLE $1
+    book setword SURVEY_STATICSKYSINGLE $1 LABEL $2
+    book setword SURVEY_STATICSKYSINGLE $1 WORKDIR $3
+    book setword SURVEY_STATICSKYSINGLE $1 DIST_GROUP $4
+    book setword SURVEY_STATICSKYSINGLE $1 SELECTDATAGROUP $5
+    book setword SURVEY_STATICSKYSINGLE $1 FILTER $6
+    book setword SURVEY_STATICSKYSINGLE $1 STATE PENDING
+end
+
+macro survey.show.staticskysingle
+ if ($0 != 1)
+    echo "USAGE: survey.show.staticskysingle"
+    break
+ end
+ book listbook SURVEY_STATICSKYSINGLE
+end
+
+macro survey.del.staticskysingle
+  if ($0 != 2)
+    echo "USAGE: survey.del.staticskysingle (tag)"
+    break
+  end
+  book delpage SURVEY_STATICSKYSINGLE $1
+end
 
 task survey.diff
@@ -682 +730 @@
     book setword SURVEY_MAGIC $label STATE DONE
     book getword SURVEY_MAGIC $label WORKDIR -var workdir
-  
-    $run = magictool -rerun -definebyquery -label $label -diff_label $label -workdir $workdir/$label
+    book getword SURVEY_MAGIC $label MULTIDIFF -var multidiff
+
+    $run = magictool -pretend -rerun -definebyquery -label $label -diff_label $label -workdir $workdir/$label
+    
+    if (("$multidiff" == "TRUE")||("$multidiff" == "MULTIDIFF"))
+       $run = $run -multidiff
+    end
 
     if ($DB:n == 0)
@@ -692 +745 @@
     end
     
-    # echo $run
+    echo $run
     command $run
   end
@@ -736 +789 @@
     endif
 
-    book getpage SURVEY_ADDSTAR 0 -var label -key STATE NEW
-    if ("$label" == "NULL")
+    book getpage SURVEY_ADDSTAR 0 -var tag -key STATE NEW
+    if ("$tag" == "NULL")
       # All labels have been done --- reset
-#      echo "Resetting labels"
+    # echo "Resetting labels"
       for i 0 $N
-        book getpage SURVEY_ADDSTAR $i -var label
-        book setword SURVEY_ADDSTAR $label STATE NEW
+        book getpage SURVEY_ADDSTAR $i -var tag
+        book setword SURVEY_ADDSTAR $tag STATE NEW
       end
-      book getpage SURVEY_ADDSTAR 0 -var label -key STATE NEW
+      book getpage SURVEY_ADDSTAR 0 -var tag -key STATE NEW
 
       # Select different database
@@ -751 +804 @@
     end
 
-    book setword SURVEY_ADDSTAR $label STATE DONE
-    book getword SURVEY_ADDSTAR $label DVODB -var dvodb
-    book getword SURVEY_ADDSTAR $label MINIDVODB_GROUP -var minidvodb_group
-    book getword SURVEY_ADDSTAR $label STAGE -var stage
+    book setword SURVEY_ADDSTAR $tag STATE DONE
+    book getword SURVEY_ADDSTAR $tag LABEL -var label
+    book getword SURVEY_ADDSTAR $tag DVODB -var dvodb
+    book getword SURVEY_ADDSTAR $tag MINIDVODB_GROUP -var minidvodb_group
+    book getword SURVEY_ADDSTAR $tag STAGE -var stage
     
   
@@ -768 +822 @@
     $run = $run -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group -stage $stage
     # echo $run
-    if ($stage == 'cam') 
+    if ("$stage" == "cam") 
         #only queue destreaked cams. stacks and staticsky don't need this
-        $run = $run -destreaked
-    end
-    if ($stage == 'staticsky') 
+        if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
+            #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked
+           $run = $run -uncensored
+        else
+           #if not isp, then run destreaked version only
+           $run = $run -destreaked
+        end
+    end
+    if ("$stage" == "staticsky") 
         #only queue uncensored staticsky
         $run = $run -uncensored
     end
-    if ($stage == 'stack') 
+    if ("$stage" == "stack") 
         #only queue uncensored stacks
         $run = $run -uncensored
@@ -784 +844 @@
     
 
-
+    echo $run
     command $run
   end
 
   # success
+   
   task.exit    0
 #    echo "Success"
@@ -1180 +1241 @@
   end
 end
+
+
+
+task survey.staticskysingle
+  host local
+
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.staticskysingle.log
+  stderr $LOGDIR/survey.staticskysingle.log
+
+    task.exec
+    book npages SURVEY_STATICSKYSINGLE -var N
+    if ($N == 0)
+      echo "No STATICSKYSINGLE labels for processing"
+       break
+    endif
+    book getpage SURVEY_STATICSKYSINGLE 0 -var tag -key STATE NEW
+    if ("$tag" == "NULL")
+      # All tags have been done --- reset
+      # echo "Resetting tags"
+      for i 0 $N
+        book getpage SURVEY_STATICSKYSINGLE $i -var tag
+        book setword SURVEY_STATICSKYSINGLE $tag STATE NEW
+      end
+      book getpage SURVEY_STATICSKYSINGLE 0 -var tag -key STATE NEW
+      
+      # Select different database
+      $SURVEY_STATICSKYSINGLE_DB ++
+      if ($SURVEY_STATICSKYSINGLE_DB >= $DB:n) set SURVEY_STATICSKYSINGLE_DB = 0
+    end
+
+    book setword SURVEY_STATICSKYSINGLE $tag STATE DONE
+    book getword SURVEY_STATICSKYSINGLE $tag LABEL -var label
+    book getword SURVEY_STATICSKYSINGLE $tag WORKDIR -var workdir
+    book getword SURVEY_STATICSKYSINGLE $tag DIST_GROUP -var dist_group
+    book getword SURVEY_STATICSKYSINGLE $tag SELECTDATAGROUP -var selectdatagroup
+    book getword SURVEY_STATICSKYSINGLE $tag FILTER -var filter
+
+    $year = `date +%Y`
+    $month = `date +%m`
+    $day = `date +%d`
+
+    $run = staticskytool -definebyquery
+    $run = $run -set_workdir $workdir/$label/$year/$month/$day
+    $run = $run -set_dist_group $dist_group
+    $run = $run -set_label $label
+    $run = $run -select_label $label
+    $run = $run -select_data_group $selectdatagroup
+    $run = $run -select_filter $filter
+#   $run = $run -pretend -simple
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_STATICSKYSINGLE_DB
+      option $DB:$SURVEY_STATICSKYSINGLE_DB
+    end
+
+#    echo $run
+
+    command $run
+  end
+
+  # success
+     task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+    task.exit    default
+    showcommand failure
+  end
+
+    task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+    task.exit    timeout
+    showcommand timeout
+  end
+end
+

tags/ipp-20110622/ippTools/share/Makefile.am

@@ -244 +244 @@
         magictool_definebyquery_insert.sql \
         magictool_definebyquery_select.sql \
+        magictool_definebyquery_select_multidiff.sql \
         magictool_inputs.sql \
         magictool_inputskyfile.sql \

tags/ipp-20110622/ippTools/share/magictool_definebyquery_select.sql

@@ -49 +49 @@
     ) AS oldMagicRun
     ON oldMagicRun.exp_id = chipRun.exp_id
---    AND oldMagicRun.diff_id = diffWarps.diff_id
 -- WHERE hook %s
 GROUP BY chipRun.exp_id

tags/ipp-20110622/ippTools/src/magictool.c

@@ -131 +131 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
-
+    PXOPT_LOOKUP_BOOL(multidiff, config->args, "-multidiff", false);
     psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
     PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
@@ -142 +142 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select.sql");
+        psString query;
+        if (multidiff) {
+          query = pxDataGet("magictool_definebyquery_select_multidiff.sql");
+        }
+        else {
+          query = pxDataGet("magictool_definebyquery_select.sql");
+        }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

tags/ipp-20110622/ippTools/src/magictoolConfig.c

@@ -61 +61 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-multidiff", 0, "allow multiple diffs to be magicked with the same label", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);