Changeset 37082

Timestamp:

Jul 17, 2014, 2:44:48 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk (updates from ops tag ipp-20130712)

Location:

tags/ipp-pv3-20140717

Files:

• . (modified) (1 prop)
• Ohana/src/getstar (modified) (1 prop)
• Ohana/src/opihi/pcontrol/PclientCommand.c (modified) (5 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (2 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 diff)
• ippScripts/scripts/lap_science.pl (modified) (1 diff)
• ippScripts/scripts/queuestaticsky.pl (modified) (1 prop)
• ippScripts/scripts/register_imfile.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (1 diff)
• ippTasks/diff.pro (modified) (2 diffs)
• ippTasks/ipphosts.mhpcc.config (modified) (2 diffs)
• ippTasks/pantasks.pro (modified) (3 diffs)
• ippTasks/stack.pro (modified) (1 diff)
• ippTasks/summit.copy.pro (modified) (1 diff)
• ippTasks/survey.pro (modified) (8 diffs)
• ippTasks/warp.pro (modified) (2 diffs)
• ippTools/share/releasetool_definerelexp.sql (modified) (1 diff)
• ippTools/src/releasetool.c (modified) (1 diff)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1/psastro.config (modified) (1 diff)
• ippconfig/recipes/filerules-split.mdc (modified) (2 diffs)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (2 diffs, 1 prop)
• ppSub/src/ppSubMatchPSFs.c (modified) (9 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 prop)
• pstamp/scripts/pstamp_finish.pl (modified) (1 prop)
• pstamp/scripts/pstamp_insert_request.pl (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (1 prop)

tags/ipp-pv3-20140717/Ohana/src/opihi/pcontrol/PclientCommand.c

@@ -25 +25 @@
     return (PCLIENT_DOWN);
   }
-  
+
   // prepare host to accept response
   host[0].response_state = response_state;
@@ -35 +35 @@
   return (PCLIENT_GOOD);
 }
-  
+
 // check for response; message must end with specified string.
 // accumulate the response in the buffer
@@ -47 +47 @@
   ASSERT (response != NULL, "response missing");
   ASSERT (buffer != NULL, "buffer missing");
+
+  // INITTIME;
 
   /* avoid blocking very long on read, test every 100 usec, up to 0.1 sec */
@@ -66 +68 @@
     return (PCLIENT_DOWN);
   }
-  if (line == NULL) return (PCLIENT_HUNG);
-  if (status == -1) return (PCLIENT_HUNG);
+  if (line == NULL) {
+      // MARKTIME ("-- client hung (line NULL): %s : %f sec\n", host[0].hostname, dtime);
+      return (PCLIENT_HUNG);
+  }
+  if (status == -1) {
+      // MARKTIME ("-- client hung (status -1): %s : %f sec\n", host[0].hostname, dtime);
+      return (PCLIENT_HUNG);
+  }
 
   // fprintf (stderr, "response: %s\n", buffer[0].buffer);
@@ -75 +83 @@
 
 /* memstr returns a view, not an allocated string : don't free */
-/* ReadtoIOBuffer returns : 
+/* ReadtoIOBuffer returns :
     0 - pipe closed
    -1 - no more data in pipe, data not ready

tags/ipp-pv3-20140717/ippScripts/scripts/dist_bundle.pl

@@ -67 +67 @@
                       );
 my %empty_cleaned = ();
-
-
 
 # Look for programs we need
@@ -496 +494 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary' or $stage eq 'ff') {
+    } elsif ($stage eq "sky" or $stage eq "skycal" or $stage eq "stack_summary" or $stage eq "ff") {
         $type = $empty_cleaned{$rule};
     } else {

tags/ipp-pv3-20140717/ippScripts/scripts/ipp_apply_burntool_single.pl

@@ -45 +45 @@
     defined $dbname;
 
-# unless (defined $camera) {
-#    $camera = "GPC1";
-# }
+# XXX watch out for TC3 and GPC2 here:
+unless (defined $camera) {
+    if ($this_uri =~ /ota/) {
+        $camera = "GPC1";
+    }
+ }
 
 my $missing_tools;

tags/ipp-pv3-20140717/ippScripts/scripts/lap_science.pl

@@ -189 +189 @@
     $chip_cmd .= " -set_label $exposure->{label} -set_data_group $data_group ";
     $chip_cmd .= " -set_workdir $workdir -set_dist_group $exposure->{dist_group} ";
-    $chip_cmd .= " -set_reduction LAP_SCIENCE ";
+#    $chip_cmd .= " -set_reduction LAP_SCIENCE ";
+    $chip_cmd .= " -set_reduction LAP_POLE ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

tags/ipp-pv3-20140717/ippScripts/scripts/register_imfile.pl

@@ -348 +348 @@
         $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
         $apply_command .= " --verbose " if $verbose;
+
+        # TEMPORARY
+        $apply_command .= " --camera GPC1";
         print "$apply_command\n";
         unless ($no_update) {

tags/ipp-pv3-20140717/ippScripts/scripts/warp_skycell.pl

@@ -413 +413 @@
                 unless ($success) {
                     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                    warn("Unable to perform warptool -addwarped: $error_code\n");
+                    warn("Unable to perform warptool -tofullskyfile: $error_code\n");
                     exit($error_code) if $try >= 2;
                     sleep 10;

tags/ipp-pv3-20140717/ippTasks/diff.pro

@@ -61 +61 @@
   end
   task diff.revert
+    active false
+  end
+end
+
+macro diff.summary.on
+  task diff.summary.load
+    active true
+  end
+  task diff.summary.run
+    active true
+  end
+end
+
+macro diff.summary.off
+  task diff.summary.load
+    active false
+  end
+  task diff.summary.run
     active false
   end
@@ -125 +143 @@
     add_poll_args run
     add_poll_labels run
+    # increase the poll limit for warp over the default to
+    # help it keep up with chip processing
+    # NOTE : it is not a problem for difftool to have multiple 
+    # -limit entries: the last one is used
+    $run = $run -limit {$POLL_LIMIT * 2}
     command $run
   end

tags/ipp-pv3-20140717/ippTasks/ipphosts.mhpcc.config

@@ -18 +18 @@
 ipphosts METADATA
   camera STR skycell
-  count S32  30
+  count S32  60
   sky00 STR  stsci00.0
   sky01 STR  stsci00.1
@@ -49 +49 @@
   sky28 STR  stsci09.1
   sky29 STR  stsci09.2
+  sky30 STR  stsci10.0
+  sky31 STR  stsci10.1
+  sky32 STR  stsci10.2
+  sky33 STR  stsci11.0
+  sky34 STR  stsci11.1
+  sky35 STR  stsci11.2
+  sky36 STR  stsci12.0
+  sky37 STR  stsci12.1
+  sky38 STR  stsci12.2
+  sky39 STR  stsci13.0
+  sky40 STR  stsci13.1
+  sky41 STR  stsci13.2
+  sky42 STR  stsci14.0
+  sky43 STR  stsci14.1
+  sky44 STR  stsci14.2
+  sky45 STR  stsci15.0
+  sky46 STR  stsci15.1
+  sky47 STR  stsci15.2
+  sky48 STR  stsci16.0
+  sky49 STR  stsci16.1
+  sky50 STR  stsci16.2
+  sky51 STR  stsci17.0
+  sky52 STR  stsci17.1
+  sky53 STR  stsci17.2
+  sky54 STR  stsci18.0
+  sky55 STR  stsci18.1
+  sky56 STR  stsci18.2
+  sky57 STR  stsci19.0
+  sky58 STR  stsci19.1
+  sky59 STR  stsci19.2
 END
 

tags/ipp-pv3-20140717/ippTasks/pantasks.pro

@@ -645 +645 @@
     host anyhost
   else
+    # echo host $host
     host $host
   end
@@ -655 +656 @@
   end
 
-  local skycellID varname count host skyname skyhash
+  local skycellID varname count host skyname skyhash fullhost
   $skycellID = $1
 
@@ -676 +677 @@
 
   # missing ipphosts table results in host = NULL
-  book getword ipphosts skycell $skyname -var host
+  book getword ipphosts skycell $skyname -var fullhost
+  list word -splitbychar . $fullhost
+  $host = $word:0
 
   if ("$host" == "NULL")
     host anyhost
   else
+    # echo host $host
     host $host
   end

tags/ipp-pv3-20140717/ippTasks/stack.pro

@@ -72 +72 @@
     active false
   end
+  task stack.summary.load
+    active false
+  end
+  task stack.summary.run
+    active false
+  end
+end
+
+macro stack.summary.on
+  task stack.summary.load
+    active true
+  end
+  task stack.summary.run
+    active true
+  end
+end
+
+macro stack.summary.off
   task stack.summary.load
     active false

tags/ipp-pv3-20140717/ippTasks/summit.copy.pro

@@ -567 +567 @@
         command pztool -clearcommonfaults -dbname $DB:$pztoolClearFault_DB
 
+        # loop over the multiple databases quickly, then pause for 5 min
         $pztoolClearFault_DB ++
-        if ($pztoolClearFault_DB >= $DB:n) set pztoolClearFault_DB = 0
+        if ($pztoolClearFault_DB >= $DB:n) 
+          set pztoolClearFault_DB = 0
+          periods -exec 300
+        else
+          periods -exec 30
+        end
       end
-      periods -exec 1800
     end
 

tags/ipp-pv3-20140717/ippTasks/survey.pro

@@ -668 +668 @@
     end
     
-#    echo $run
     command $run
   end
@@ -752 +751 @@
     end
     
-#    echo $run
     command $run
   end
@@ -838 +836 @@
     end
     
-    # echo $run
     command $run
   end
@@ -1437 +1434 @@
   host local
  
-  periods      -poll $SURVEY_POLL
-  periods      -exec $SURVEY_EXEC
+  # we are going to use a long timeout (900 sec) between passes of the list
+  # and a short timeout (10 sec) while we are doing the full list
+  periods      -exec 10
+  periods      -poll 2
   periods      -timeout $SURVEY_TIMEOUT
   npending     1
@@ -1449 +1448 @@
     book npages SURVEY_PUBLISH -var N
     if ($N == 0)
-    #  echo "No labels for processing"
+      # echo "No labels for processing"
       break
     endif
+
+    # short timeout while we have something to do
+    periods -exec 10
 
     # survey.publish allows multiple entries per label.
@@ -1465 +1467 @@
       book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
 
-      # Select different database
-      $SURVEY_PUBLISH ++
-      if ($SURVEY_PUBLISH >= $DB:n) set SURVEY_PUBLISH = 0
+      # Select different database, set a long timeout
+      $SURVEY_PUBLISH_DB ++
+      if ($SURVEY_PUBLISH_DB >= $DB:n) set SURVEY_PUBLISH_DB = 0
+      periods -exec 900
+      date -var mytime
+      echo "done with publish list @ $mytime"
     end
 
@@ -1484 +1489 @@
       option DEFAULT
     else
-      $run = $run -dbname $DB:$SURVEY_PUBLISH
-      option $DB:$SURVEY_PUBLISH
+      $run = $run -dbname $DB:$SURVEY_PUBLISH_DB
+      option $DB:$SURVEY_PUBLISH_DB
     end
     
     # echo $run
-
     command $run
     
@@ -1859 +1863 @@
     end
     
-     echo $run
     command $run
   end

tags/ipp-pv3-20140717/ippTasks/warp.pro

@@ -99 +99 @@
   end
   task warp.revert.overlap
+    active false
+  end
+end
+
+macro warp.summary.on
+  task warp.summary.load
+    active true
+  end
+  task warp.summary.run
+    active true
+  end
+end
+
+macro warp.summary.off
+  task warp.summary.load
+    active false
+  end
+  task warp.summary.run
     active false
   end
@@ -261 +279 @@
     add_poll_args run
     add_poll_labels run
+    # increase the poll limit for warp over the default to
+    # help it keep up with chip processing
+    # NOTE : it is not a problem for warptool to have multiple 
+    # -limit entries: the last one is used
+    $run = $run -limit {$POLL_LIMIT * 2}
     command $run
   end

tags/ipp-pv3-20140717/ippTools/share/releasetool_definerelexp.sql

@@ -18 +18 @@
 
 WHERE previousrelexp.relexp_id IS NULL
+    AND camRun.state = 'full'

tags/ipp-pv3-20140717/ippTools/src/releasetool.c

@@ -780 +780 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC");
+        psStringAppend(&query, "\nAND priority > 0 ORDER BY exp_id, priority DESC");
+    } else {
+        psStringAppend(&query, "\nORDER BY exp_id, relexp_id DESC");
     }
 

tags/ipp-pv3-20140717/ippconfig/gpc1/psastro.config

@@ -388 +388 @@
 
 POLE_REFCAT METADATA
-   PSASTRO.CATDIR               STR  SYNTH.GRIZY
+   PSASTRO.CATDIR               STR  PS1.REF.20140713
    ZERO.POINT.USE.MEAN               BOOL TRUE  
    PSASTRO.GRID.MIN.ANGLE       F32  -20

tags/ipp-pv3-20140717/ippconfig/recipes/filerules-split.mdc

@@ -306 +306 @@
                                                                                                               
 PPSUB.OUTPUT                 OUTPUT {OUTPUT}.fits                     IMAGE           NONE       FPA        TRUE      NONE
-PPSUB.OUTPUT.MASK            OUTPUT {OUTPUT}.mask.fits                MASK            COMP_MASK  FPA        TRUE      NONE
-PPSUB.OUTPUT.VARIANCE        OUTPUT {OUTPUT}.wt.fits                  VARIANCE        COMP_WT    FPA        TRUE      NONE
+PPSUB.OUTPUT.MASK            OUTPUT {OUTPUT}.mask.fits                MASK            NONE       FPA        TRUE      NONE
+PPSUB.OUTPUT.VARIANCE        OUTPUT {OUTPUT}.wt.fits                  VARIANCE        NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.SOURCES         OUTPUT {OUTPUT}.cmf                      CMF             NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.KERNELS         OUTPUT {OUTPUT}.subkernel                SUBKERNEL       NONE       FPA        TRUE      NONE
@@ -314 +314 @@
 PPSUB.OUTPUT.RESID.JPEG      OUTPUT {OUTPUT}.res.jpg                  JPEG            NONE       FPA        TRUE      NONE
 PPSUB.INVERSE                OUTPUT {OUTPUT}.inv.fits                 IMAGE           NONE       FPA        TRUE      NONE
-PPSUB.INVERSE.MASK           OUTPUT {OUTPUT}.inv.mask.fits            MASK            COMP_MASK  FPA        TRUE      NONE
-PPSUB.INVERSE.VARIANCE       OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE        COMP_WT    FPA        TRUE      NONE
+PPSUB.INVERSE.MASK           OUTPUT {OUTPUT}.inv.mask.fits            MASK            NONE       FPA        TRUE      NONE
+PPSUB.INVERSE.VARIANCE       OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE        NONE       FPA        TRUE      NONE
 PPSUB.INVERSE.SOURCES        OUTPUT {OUTPUT}.inv.cmf                  CMF             NONE       FPA        TRUE      NONE
 PPSUB.CONFIG                 OUTPUT {OUTPUT}.ppSub.mdc                TEXT            NONE       FPA        TRUE      NONE

tags/ipp-pv3-20140717/ippconfig/recipes/reductionClasses.mdc

@@ -244 +244 @@
         ADDSTAR           STR     ADDSTAR
         PSASTRO           STR     LAP_ASTRO
+        STACKPHOT_PSPHOT  STR     STACKPHOT
+        STACKPHOT_PPSUB   STR     STACKPHOT
+        STACKPHOT_PPSTACK STR     STACKPHOT
+        STACKPHOT_SINGLE_PSPHOT  STR     STACKPHOT_SINGLE
+        BACKGROUND_PPBACKGROUND STR     BACKGROUND
+        BACKGROUND_PSWARP       STR     BACKGROUND
+        FULLFORCE_PSPHOT  STR   FULLFORCE_WARP
+END
+LAP_POLE                METADATA
+        CHIP_PPIMAGE      STR     CHIP
+        CHIP_PSPHOT       STR     CHIP
+        WARP_PSWARP       STR     WARP
+        STACK_PPSTACK     STR     STACK
+        STACK_PPSUB       STR     STACK
+        STACK_PSPHOT      STR     STACK
+        DIFF_PPSUB        STR     DIFF
+        DIFF_PSPHOT       STR     DIFF
+        JPEG_BIN1         STR     PPIMAGE_J1
+        JPEG_BIN2         STR     PPIMAGE_J2
+        FAKEPHOT          STR     FAKEPHOT
+        ADDSTAR           STR     ADDSTAR
+        PSASTRO           STR     POLE_REFCAT
         STACKPHOT_PSPHOT  STR     STACKPHOT
         STACKPHOT_PPSUB   STR     STACKPHOT
@@ -1172 +1194 @@
 
 STS_DATASET_NOCOMPRESS METADATA
+        # XXX: don't use this recipe
    CHIP_PPIMAGE STR CHIP_NOCOMPRESS
    CHIP_PSPHOT STR CHIP

tags/ipp-pv3-20140717/ppSub/src/ppSubMatchPSFs.c

@@ -167 +167 @@
     }
 
-    // we need to register the FWHM values for use downstream 
+    // we need to register the FWHM values for use downstream
     pmSubtractionSetFWHMs(inFWHM, refFWHM);
 
@@ -220 +220 @@
     bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images.
     if (noConvolve) {
-        psWarning("not matching PSFs because NOCONVOLVE is TRUE\n");
-        return true;
+        psWarning("not matching PSFs because NOCONVOLVE is TRUE\n");
+        return true;
     }
 
@@ -235 +235 @@
         pmCell *cell = pmFPAfileThisCell(config->files, view, "PPSUB.INPUT.CONV"); // Cell for convolved input
         inConv = pmReadoutAlloc(cell);
-        psFree(inConv);
+        psFree(inConv);
     }
     pmReadout *refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference convolved
@@ -241 +241 @@
         pmCell *cell = pmFPAfileThisCell(config->files, view, "PPSUB.REF.CONV"); // Cell for convolved ref.
         refConv = pmReadoutAlloc(cell);
-        psFree(refConv);
+        psFree(refConv);
     }
 
@@ -269 +269 @@
         lists->data[0] = psMemIncrRefCounter(inSources);
         lists->data[1] = psMemIncrRefCounter(refSources);
-        // XXX MEH changed to get only match (true to cull single), no apparent need of unmatched sources and can cause trouble
+        // XXX MEH changed to get only match (true to cull single), no apparent need of unmatched sources and can cause trouble
         detections->allSources = pmSourceMatchMerge(lists, radius, true);
         psFree(lists);
@@ -365 +365 @@
         subMode = PM_SUBTRACTION_MODE_DUAL;
     } else {
-        char *convolveName = psMetadataLookupStr(&mdok, recipe, "CONVOLVE.TARGET"); // recipe value for target
+        char *convolveName = psMetadataLookupStr(&mdok, recipe, "CONVOLVE.TARGET"); // recipe value for target
         int convolve = psMetadataLookupS32(&mdok, config->arguments, "-convolve"); // override with command-line option
-        switch (convolve) {
-          case 0:
-            // convolve is 0 if it is not supplied on the command line
-            subMode = subModeFromString(convolveName);
-            break;
-          case 1:
-            subMode = PM_SUBTRACTION_MODE_1;
-            break;
-          case 2:
-            subMode = PM_SUBTRACTION_MODE_2;
-            break;
-          default:
-            psError(PPSUB_ERR_ARGUMENTS, false, "Invalid value for -convolve");
-            return false;
-        }
+        switch (convolve) {
+          case 0:
+            // convolve is 0 if it is not supplied on the command line
+            subMode = subModeFromString(convolveName);
+            break;
+          case 1:
+            subMode = PM_SUBTRACTION_MODE_1;
+            break;
+          case 2:
+            subMode = PM_SUBTRACTION_MODE_2;
+            break;
+          default:
+            psError(PPSUB_ERR_ARGUMENTS, false, "Invalid value for -convolve");
+            return false;
+        }
     }
 
@@ -397 +397 @@
         psKernel *truncated = psImageCovarianceTruncate(inRO->covariance, COVAR_FRAC);
         psFree(inRO->covariance);
-        inRO->covariance = truncated;
+        inRO->covariance = truncated;
     }
     if (refRO->covariance) {
         psKernel *truncated = psImageCovarianceTruncate(refRO->covariance, COVAR_FRAC);
         psFree(refRO->covariance);
-        refRO->covariance = truncated;
+        refRO->covariance = truncated;
     }
 
@@ -458 +458 @@
             ppSubDataQuality(data, error, PPSUB_FILES_ALL);
             return true;
+        // XX } else if (error == PM_ERR_DATA) {
+        // XX     psErrorStackPrint(stderr, "Unable to solve for the kernel to match images");
+        // XX     psWarning("Failed to find PSF match kernel --- suspect bad data quality.");
+        // XX     ppSubDataQuality(data, error, PPSUB_FILES_ALL);
+        // XX     return true;
         } else {
             psError(PPSUB_ERR_DATA, false, "Unable to match images.");
@@ -515 +520 @@
         psKernel *truncated = psImageCovarianceTruncate(inConv->covariance, COVAR_FRAC);
         psFree(inConv->covariance);
-        inConv->covariance = truncated;
+        inConv->covariance = truncated;
     }
     if (refConv->covariance) {
         psKernel *truncated = psImageCovarianceTruncate(refConv->covariance, COVAR_FRAC);
         psFree(refConv->covariance);
-        refConv->covariance = truncated;
+        refConv->covariance = truncated;
     }
 

tags/ipp-pv3-20140717/pstamp/scripts/pstamp_insert_request.pl

@@ -119 +119 @@
 }
 
-my $datestr = strftime "%Y%m%d", gmtime;
+# put file in directory for the current date
+my $datestr = strftime "%Y/%m/%d", gmtime;
 my $datedir = "$workdir/webreq/$datestr";
 if (! -e $datedir ) {
-    if (!  mkdir $datedir ) {
-        print STDERR  "failed to create working directory $datedir";
+    my $rc = system "mkdir -p $datedir";
+    if ($rc) {
+        my $status = $rc >> 8;
+        print STDERR  "failed to create working directory $datedir: $rc $status";
         exit $PS_EXIT_CONFIG_ERROR;
     }
 }
+
 
 my $webreq_num = get_webreq_num();