Index: trunk/ippTasks/phase0.pro
===================================================================
--- trunk/ippTasks/phase0.pro	(revision 10746)
+++ trunk/ippTasks/phase0.pro	(revision 11210)
@@ -1,11 +1,5 @@
 ## this file contains the tasks for running the phase 0 stage
-
-## these tasks use the queue Phase0Imfiles
-## the Phase0Imfiles queue contains:
-## STATE EXP_TAG CLASS CLASS_ID URI-FITS
-## queue keys are counted from 0
-
-queueinit Phase0Imfiles
-queueinit Phase0Exposures
+## these tasks use the books p0PendingImfile and p0PendingExp
+
 if ($?network == 0)
   $network = 1
@@ -14,8 +8,4 @@
   $parallel = 0
 end
-
-$Phase0ImfileFail = 0
-$Phase0ExpFail = 0
-
 if ($?VERBOSE == 0)
   echo "VERBOSE not defined: load pantasks.pro first"
@@ -26,16 +16,20 @@
   break
 end
+
 $LOGSUBDIR = $LOGDIR/phase0
 exec mkdir -p $LOGSUBDIR
 
+book init p0PendingImfile
+book init p0PendingExp
+
 macro phase0.status
-  queueprint Phase0Imfiles
-  queueprint Phase0Exposures
-  echo "Phase0ImfileFail: $Phase0ImfileFail"
-  echo "Phase0ExpFail: $Phase0ExpFail"
-end
+  book listbook p0PendingImfile
+  book listbook p0PendingExp
+end
+
+$phase0_DB = 0
 
 # select images ready for phase0 analysis
-# new entries are added to queue Phase0Imfiles
+# new entries are added to p0PendingImfile
 # compare the new list with the ones already selected
 task	       phase0.imfile.load
@@ -46,33 +40,34 @@
   periods      -timeout 30
 
-  stdout $LOGSUBDIR/phase0.imfile.log
+  # XXX test that this option works
+  npending     1
+
+  # silently drop stdout
+  stdout NULL
   stderr $LOGSUBDIR/phase0.imfile.log
 
-  task.exec
-    command p0tool -pendingimfile -simple -limit {$Phase0ImfileFail + 20}
+  # select entries from the current DB; cycle to the next DB, if it exists
+  # if the DB list is not set, use the value defined in .ipprc
+  task.exec
+    if ($DB:n == 0)
+      command p0tool -pendingimfile -limit 20
+    else
+      command p0tool -pendingimfile -limit 20 -dbname $DB:$phase0_DB
+      $phase0_DB ++
+      if ($phase0_DB >= $DB:n) set phase0_DB = 0
+    end
   end
 
   # success
-  task.exit    0
-    local i Nqueue
-
-    # compare output with Phase0Imfiles queue
-    # only add entries which don't exist in queue
-    queuesize stdout -var Nqueue
-    for i 0 $Nqueue
-      queuepop stdout -var line
-      if ($VERBOSE > 2)
-        echo $line
-      end
-      list word -split $line
-      $EXP_TAG  = $word:0
-      $CLASS    = $word:1
-      $CLASS_ID = $word:2
-      $URI      = $word:3
-      queuepush Phase0Imfiles -uniq -key 1 "NEW $EXP_TAG $CLASS $CLASS_ID $URI"
-    end
-
-    # delete existing entries which are DONE
-    queuedrop Phase0Imfiles -key 0 DONE
+  task.exit $EXIT_SUCCESS
+    # convert 'stdout' to book format
+    ipptool2book stdout p0PendingImfile -key exp_tag -uniq
+    if ($VERBOSE > 2)
+      book listbook p0PendingImfile
+    end
+
+    # delete existing entries which are DONE or DATAERR
+    book delpage p0PendingImfile -key state DONE
+    book delpage p0PendingImfile -key state DATA_ERR
   end
 
@@ -88,7 +83,5 @@
 end
 
-
-
-# run the phase0imfile script on pending images
+# run the phase0_imfile.pl script on pending images
 task	       phase0.imfile.run
   periods      -poll $runpoll
@@ -97,24 +90,114 @@
 
   task.exec
-    queuesize Phase0Imfiles -var N
+    book npages p0PendingImfile -var N
     if ($N == 0) break
     if ($network == 0) break
     
-    # look for new images on the internal DetrendImfilesToProcess queue
-    # caution with these 'if' statements: syntax errors 
-    # will make the task fail without given a good status
-    queuepop Phase0Imfiles -var line -key 0 NEW
-    if ("$line" == "NULL") break
-
-    strpop line state
-    queuepush Phase0Imfiles -replace -key 1 "RUN $line"
-
-    ## the Phase0Imfiles queue contains (line has popped STATE)
-    ## STATE EXP_TAG CLASS CLASS_ID URI-FITS
-    list word -split $line
-    $EXP_TAG  = $word:0
-    $CLASS    = $word:1
-    $CLASS_ID = $word:2
-    $URI      = $word:3
+    # look for new images in p0PendingImfile
+    book getpage p0PendingImfile 0 -var pageName -key state NULL
+    if ($pageName == NULL) break
+
+    book setword p0PendingImfile $pageName state RUN
+
+    book getword p0PendingImfile $pageName exp_tag -var EXP_TAG
+    book getword p0PendingImfile $pageName class -var CLASS
+    book getword p0PendingImfile $pageName class_id -var CLASS_ID
+    book getword p0PendingImfile $pageName uri -var URI
+    book getword p0PendingImfile $pageName outpath -var OUTPATH
+    book getword p0PendingImfile $pageName dbname -var DBNAME
+
+    # specify choice of remote host
+    if ($parallel)
+      host anyhost
+    else
+      host local
+    end
+
+    ## generate output log based on filerule
+    $outroot = $OUTPATH/$EXP_TAG
+    $logfile = `ipp_filename.pl PHASE0.LOG $outroot $CLASS_ID`
+    stdout $logfile
+    stderr $logfile
+    exec mkdir -p $OUTPATH
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME
+    end
+    command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI --dbname $DBNAME
+  end
+
+  # default exit status
+  task.exit default
+    process_exit p0PendingImfile $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword p0PendingImfile $options:0 state TIMEOUT
+  end
+end
+
+# select exposures ready for phase0_exp.pl
+task	       phase0.exp.load
+  host         local
+
+  periods      -poll $loadpoll
+  periods      -exec $loadexec
+  periods      -timeout 30
+
+  stdout NULL
+  stderr $LOGSUBDIR/phase0.exp.log
+
+  task.exec
+    command p0tool -pendingexp -limit 20
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout p0PendingExp -key exp_tag -uniq
+    if ($VERBOSE > 2)
+      book listbook p0PendingExp
+    end
+
+    # delete existing entries which are DONE
+    book delpage p0PendingExp -key state DONE
+    book delpage p0PendingExp -key state DATA_ERR
+  end
+
+  # default exit status
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+# run the phase0_exp.pl script on pending exposures
+task	       phase0.exp.run
+  periods      -poll $runpoll
+  periods      -exec $runexec
+  periods      -timeout 30
+
+  task.exec
+    book npages p0PendingExp -var N
+    if ($N == 0) break
+    if ($network == 0) break
+    
+    # look for new images in p0PendingExp (state == NULL)
+    book getpage p0PendingExp 0 -var pageName -key state NULL
+    if ($pageName == NULL) break
+
+    book setword p0PendingExp $pageName state RUN
+    book getword p0PendingExp $pageName exp_tag -var EXP_TAG
+    book getword p0PendingExp $pageName outpath -var OUTPATH
 
     # specify choice of remote host:(need to choose based on chips)
@@ -125,146 +208,15 @@
     end
 
-    # XXX this is still a lame rule
-    $word = `basename $EXP_TAG | tr '.' ' '`
-    list word -split $word
-    $base = $word:0
-    stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log
-    stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log
-    exec mkdir -p $LOGSUBDIR/$base
+    ## generate output log based on filerule
+    $outroot = $OUTPATH/$EXP_TAG
+    $logfile = `ipp_filename.pl PHASE0.LOG $outroot FPA`
+    stdout $logfile
+    stderr $logfile
+    exec mkdir -p $OUTPATH
+
+    # save the pageName for future reference below
+    options $pageName
 
     # create the command line
-    options "$line"
-    if ($VERBOSE > 1)
-      echo command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI
-    end
-    command phase0_imfile.pl --exp_tag $EXP_TAG --class_id $CLASS_ID --uri $URI
-  end
-
-  # success
-  task.exit 0
-    # phase0imfile.pl updates DB tables, here we just update the queue
-    queuepush Phase0Imfiles -replace -key 1 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand failure
-    queuepush Phase0Imfiles -replace -key 1 "FAIL $options:0"
-    $Phase0ImfileFail ++
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-    queuepush Phase0Imfiles -replace -key 1 "TIMEOUT $options:0"
-    $Phase0ImfileFail ++
-  end
-end
-
-
-
-## these tasks use the queue Phase0Exposures
-## the Phase0Exposures queue contains:
-## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS
-
-# select images ready for phase0 analysis
-# new entries are added to queue Phase0Exposures
-# compare the new list with the ones already selected
-task	       phase0.exp.load
-  host         local
-
-  periods      -poll $loadpoll
-  periods      -exec $loadexec
-  periods      -timeout 30
-
-  stdout $LOGSUBDIR/phase0.exp.log
-  stderr $LOGSUBDIR/phase0.exp.log
-
-  task.exec
-    command p0tool -pendingexp -simple -limit {$Phase0ExpFail + 20}
-  end
-
-  # success
-  task.exit    0
-    local i Nqueue
-
-    # compare output with Phase0Exposures queue
-    # only add entries which don't exist in queue
-    queuesize stdout -var Nqueue
-    for i 0 $Nqueue
-      queuepop stdout -var line
-      if ($VERBOSE > 2)
-        echo $line
-      end
-      list word -split $line
-      $EXP_TAG   = $word:0
-      # $EXP_ID    = $word:1
-      # $CAMERA    = $word:2
-      # $TELESCOPE = $word:3
-      # $DATEOBS   = $word:4
-      # $TYPE      = $word:5
-      # $IMFILES   = $word:6
-      queuepush Phase0Exposures -uniq -key 1 "NEW $EXP_TAG"
-    end
-
-    # delete existing entries which are DONE
-    queuedrop Phase0Exposures -key 0 DONE
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand
-  end
-
-  # operation times out?
-  task.exit    timeout
-    showcommand
-  end
-end
-
-
-
-# run the phase0imfile script on pending images
-task	       phase0.exp.run
-  periods      -poll $runpoll
-  periods      -exec $runexec
-  periods      -timeout 30
-
-  task.exec
-    queuesize Phase0Exposures -var N
-    if ($N == 0) break
-    if ($network == 0) break
-    
-    # look for new images on the internal DetrendImfilesToProcess queue
-    # caution with these 'if' statements: syntax errors 
-    # will make the task fail without given a good status
-    queuepop Phase0Exposures -var line -key 0 NEW
-    if ("$line" == "NULL") break
-
-    strpop line state
-    queuepush Phase0Exposures -replace -key 1 "RUN $line"
-
-    ## the Phase0Exposures queue contains:
-    ## STATE EXP_TAG CAMERA TELESCOPE TYPE NCLASS
-    list word -split $line
-    $EXP_TAG   = $word:0
-
-    # XXX this is still a lame rule
-    $word = `basename $EXP_TAG | tr '.' ' '`
-    list word -split $word
-    $base = $word:0
-    stdout $LOGSUBDIR/$base/$EXP_TAG.p0.log
-    stderr $LOGSUBDIR/$base/$EXP_TAG.p0.log
-    exec mkdir -p $LOGSUBDIR/$base
-
-    # specify choice of remote host:(need to choose based on chips)
-    if ($parallel)
-      host anyhost
-    else
-      host local
-    end
-
-    # create the command line
-    options "$line"
     if ($VERBOSE > 1)
       echo command phase0_exp.pl --exp_tag $EXP_TAG
@@ -274,21 +226,12 @@
 
   # success
-  task.exit 0
-    # phase0exp.pl updates DB tables, here we just update the queue
-    queuepush Phase0Exposures -replace -key 1 "DONE $options:0"
-  end
-
-  # default exit status
-  task.exit    default
-    showcommand
-    queuepush Phase0Exposures -replace -key 1 "FAIL $options:0"
-    $Phase0ExpFail
+  task.exit default
+    process_exit p0PendingExp $options:0 $JOB_STATUS
   end
 
   # operation times out?
   task.exit    timeout
-    showcommand
-    queuepush Phase0Exposures -replace -key 1 "TIMEOUT $options:0"
-    $Phase0ExpFail
+    showcommand timeout
+    book setword p0PendingExp $options:0 state TIMEOUT
   end
 end