Changeset 34794

Timestamp:

Dec 11, 2012, 11:15:37 AM (14 years ago)

Author:

bills

Message:

Add n_forced column to skycalResult table. Modify the scripts and recipes
to set it.
Add some keywords to the CAMSTATS recipe use them to set n_detections, n_extended,
and n_forced.
Modify skycalibration.pl to handle the various filerules used for staticsky. Now
can find cmf for singlefilter runs whether it was created by psphot or psphotStack

Location:

tags/ipp-20121026

Files:

• dbconfig/changes.txt (modified) (1 diff, 1 prop)
• dbconfig/staticsky.md (modified) (1 diff, 1 prop)
• ippScripts/scripts/skycalibration.pl (modified) (5 diffs, 1 prop)
• ippScripts/scripts/staticsky.pl (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff, 1 prop)
• ippTools/share/staticskytool_pendingskycalrun.sql (modified) (1 prop)
• ippTools/src/staticskytool.c (modified) (8 diffs, 1 prop)
• ippTools/src/staticskytoolConfig.c (modified) (4 diffs, 1 prop)
• ippconfig/recipes/ppStats.config (modified) (1 diff, 1 prop)
• ippconfig/recipes/ppStatsFromMetadata.config (modified) (1 diff, 1 prop)

tags/ipp-20121026/dbconfig/changes.txt

@@ -2262 +2262 @@
 
 UPDATE dbversion set schema_version = '1.1.72', updated= CURRENT_TIMESTAMP();
+
+ALTER TABLE skycalResult ADD column n_forced INT after n_extended;
+UPDATE dbversion set schema_version = '1.1.73', updated= CURRENT_TIMESTAMP();

tags/ipp-20121026/dbconfig/staticsky.md

@@ -56 +56 @@
     n_detections   S32    0
     n_extended     S32    0
+    n_forced       S32    0
     zpt_obs        F32    0.0
     zpt_stdev      F32    0.0

tags/ipp-20121026/ippScripts/scripts/skycalibration.pl

@@ -134 +134 @@
 # First check the expected file name where the stack_id is the FILE_ID
 my $file;
-if (!$singlefilter) {
+# if (!$singlefilter) {
+if (1) {
+
     $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id);
     if (! $ipprc->file_resolve($file)) {
-        # XXX: Beginning of section that can be removed eventually
-
-        # no file with the expected name found
-        # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID
-        # but instead used FILE_ID = [0 .. num_filters-1]
         print "\nfailed to resolve $file\n";
-        if (!$filter) {
-            &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
-        }
-        print "Trying old style FILE_ID\n";
-        my $max_filters = 5;
-        for (my $i=0; $i < $max_filters; $i++) {
-            my $file_id = sprintf "%03d", $i;
-            $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
-            my $resolved = $ipprc->file_resolve($file);
-            if (!$resolved) {
-                # we fail here assumming that if file_id N doesn't exist, neither to N+1
-                print "\nfailed to resolve $file\n\n";
-                &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+        # file with proper file rule not found. Try older systems
+        if ($singlefilter) {
+            # input is from a single filter static sky run which used psphot instead of psphotStack
+            # The file rule is different for psphot
+            $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
+            &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) 
+                unless $ipprc->file_exists($file);
+        } else {
+            # XXX: Beginning of section that can be removed eventually
+            # no file with the expected name found
+            # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID
+            # but instead used FILE_ID = [0 .. num_filters-1]
+            if (!$filter) {
+                &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
             }
-            # Check the filter id for this file
-            my $command = "$fhead $resolved | grep FILTERID";
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => 0);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+            print "Trying old style FILE_ID\n";
+            my $max_filters = 5;
+            for (my $i=0; $i < $max_filters; $i++) {
+                my $file_id = sprintf "%03d", $i;
+                $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
+                my $resolved = $ipprc->file_resolve($file);
+                if (!$resolved) {
+                    # we fail here assumming that if file_id N doesn't exist, neither to N+1
+                    print "\nfailed to resolve $file\n\n";
+                    &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+                }
+                # Check the filter id for this file
+                my $command = "$fhead $resolved | grep FILTERID";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => 0);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+                }
+                # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
+                my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
+                my $this_filter = substr $filt, 1, 7;
+
+                # if it matches we're done
+                last if $this_filter eq $filter;
+
+                # nope loop around to try the next one
+                print "Input file for $filter is not $file ($this_filter)\n";
+                $file = undef;
             }
-            # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
-            my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
-            my $this_filter = substr $filt, 1, 7;
-
-            # if it matches we're done
-            last if $this_filter eq $filter;
-
-            # nope loop around to try the next one
-            print "Input file for $filter is not $file ($this_filter)\n";
-            $file = undef;
-        }
+        }
+        # XXX: End of section that can be removed eventually
+    }
+    if ($file) {
         print "\nInput file for $stack_id filter: $filter: $file\n";
-
-        # XXX: End of section that can be removed eventually
-    }
-} else {
-    # input is from a single filter static sky run use a different file rule
-    # XXX: can't we just make staticsky.pl use the same file rule?
-    $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
-    &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) 
-        unless $ipprc->file_exists($file);
-}
-
-if (!$file) {
-    &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR);
+    } else {
+        &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR);
+    }
 }
 
@@ -197 +200 @@
 }
 
-# read the input header to find the number of detections and the number of detections
-# with extended model
+# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET
+# read the input header to find the number of detections
+my $compatability_flags = "";
 my $n_detections = 0;
-my $n_extended = 0;
 {
+    # NSTARS is buggy. It only counts sources with a model, and counts them twice.
+    # We want the actual number of sources (real + matched). Use the length of the psf table
     my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -211 +216 @@
         unless defined $n_detections;
 
-    $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 0);
-    (undef, $n_extended)  = split " ", join "", @$stdout_buf;
-    chomp $n_extended;
-    &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR)
-        unless defined $n_extended;
+    $compatability_flags .= " -n_detections $n_detections";
+
 }
 
@@ -257 +257 @@
         }
 
+        # if ppStats didn't get n_detections use the compatability version extracted above
+        $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/);
+
         my $quality;
         ($quality) =  $cmdflags =~ /-quality (\d+)/;
@@ -276 +279 @@
 {
     my $command = "$staticskytool -skycal_id $skycal_id";
-    $command .= " -n_detections $n_detections";
-    $command .= " -n_extended $n_extended";
     $command .= " -addskycalresult -path_base $outroot";
     $command .= " $cmdflags";

tags/ipp-20121026/ippTools/share/pxadmin_create_tables.sql

@@ -2119 +2119 @@
       n_detections INT,
       n_extended INT,
+      n_forced INT,
       zpt_obs FLOAT,
       zpt_stdev FLOAT,

tags/ipp-20121026/ippTools/src/staticskytool.c

@@ -132 +132 @@
     pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "stackRun.data_group",       "LIKE");
     pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
-#ifdef notdef
-    PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min",      "skycell.glat",              ">=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max",      "skycell.glat",              "<=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min",   "skycell.radeg/15",          ">=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max",   "skycell.radeg/15",          "<=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min",     "skycell.radeg",             ">=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max",     "skycell.radeg",             "<=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min",    "skycell.decdeg",             ">=");
-    PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max",    "skycell.decdeg",             "<=");
-#endif
     if (!pxskycellAddWhere(config, whereMD)) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -686 +676 @@
     psStringAppend(&query, "\nGROUP BY sky_id");
     if (num_filters) {
-        psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters);
+        psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters);
     }
         
@@ -1049 +1039 @@
     pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "staticskyRun.data_group",       "LIKE");
     pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
+    if (!pxskycellAddWhere(config, whereMD)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
+        return false;
+    }
 
     PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
@@ -1128 +1122 @@
         psString sky_label =  psMetadataLookupStr(&status, row, "label");
         psString sky_data_group =  psMetadataLookupStr(&status, row, "data_group");
+        psString sky_dist_group =  psMetadataLookupStr(&status, row, "dist_group");
 
         // create a staticskyRun
@@ -1138 +1133 @@
                                 label ? label : sky_label,
                                 data_group ? data_group : sky_data_group,
-                                dist_group,
+                                dist_group ? dist_group : sky_dist_group,
                                 reduction,
                                 registered,
@@ -1159 +1154 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycal_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-sky_id",   "sky_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id",   "==");
     PXOPT_COPY_STR(config->args, where, "-label",   "label",    "==");
     PXOPT_COPY_STR(config->args, where, "-state",   "state",    "==");
@@ -1276 +1273 @@
     PXOPT_LOOKUP_STR(hostname, config->args,    "-hostname", false, false);
     PXOPT_LOOKUP_S32(n_astrom, config->args,    "-n_astrom", false, false);
-    PXOPT_LOOKUP_S32(n_detections, config->args,    "-n_detections", false, false);
-    PXOPT_LOOKUP_S32(n_extended, config->args,    "-n_extended", false, false);
+    PXOPT_LOOKUP_S32(n_detections, config->args,"-n_detections", false, false);
+    PXOPT_LOOKUP_S32(n_extended, config->args,  "-n_extended", false, false);
+    PXOPT_LOOKUP_S32(n_forced, config->args,    "-n_forced", false, false);
 
     PXOPT_LOOKUP_STR(path_base, config->args,   "-path_base", false, false);
@@ -1318 +1316 @@
         n_detections,
         n_extended,
+        n_forced,
         zpt_obs,
         zpt_stdev,

tags/ipp-20121026/ippTools/src/staticskytoolConfig.c

@@ -126 +126 @@
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
-    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0);
+    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0);
     pxskycellAddArguments(resultArgs);
     psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
@@ -166 +166 @@
     psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
     psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
+    pxskycellAddArguments(defineskycalrunArgs);
     psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
     psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
@@ -181 +182 @@
     psMetadata *updateskycalrunArgs = psMetadataAlloc();
     psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
+    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0,    "search by sky ID", 0);
+    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0,  "search by stack ID", 0);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
+    pxskycellAddArguments(updateskycalrunArgs);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
@@ -215 +219 @@
     psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_detections", 0,        "define number of detections", 0);
     psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_extended", 0,          "define number of extended detections", 0);
+    psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_forced", 0,            "define number of forced detections", 0);
 
     psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);

tags/ipp-20121026/ippconfig/recipes/ppStats.config

@@ -127 +127 @@
   HEADER        STR     IMAGE_V
   HEADER        STR     STREAK_V
+  
+  HEADER        STR     NDET            # sources->n
+  HEADER        STR     NDET_EXT
+  HEADER        STR     NDET_FRC
 
   ANALYSIS      MULTI   # metadata blocks to search in chip/cell/readout->analysis

tags/ipp-20121026/ippconfig/recipes/ppStatsFromMetadata.config

@@ -544 +544 @@
   ENTRY  VAL  DT_ASTR             F32  CONSTANT         -dtime_astrom      
   ENTRY  VAL  NASTRO              S32  CONSTANT         -n_astrom          
+  ENTRY  VAL  NDET                S32  CONSTANT         -n_detections
+  ENTRY  VAL  NDET_EXT            S32  CONSTANT         -n_extended
+  ENTRY  VAL  NDET_FRC            S32  CONSTANT         -n_forced
 
   ENTRY  VAL  ZPT_OBS             F32  CONSTANT         -zpt_obs