Changeset 36546

Timestamp:

Feb 25, 2014, 11:25:44 AM (12 years ago)

Author:

bills

Message:

Changes to support distribution of full force data and some
property updates from previous merges with the trunk

Location:

tags/ipp-20140114

Files:

• ippScripts/scripts/dist_bundle.pl (modified) (9 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/queuestaticsky.pl (modified) (2 diffs)
• ippTools/share/Makefile.am (modified) (3 diffs)
• ippTools/share/disttool_definebyquery_ff.sql (added)
• ippTools/share/disttool_pending_ff.sql (added)
• ippTools/share/disttool_toadvance.sql (modified) (1 diff)
• ippTools/share/fftool_export_input.sql (added)
• ippTools/share/fftool_export_result.sql (added)
• ippTools/share/fftool_export_run.sql (added)
• ippTools/share/fftool_export_summary.sql (added)
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/fftool.c (modified) (7 diffs)
• ippTools/src/fftool.h (modified) (1 diff)
• ippTools/src/fftoolConfig.c (modified) (2 diffs)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)

tags/ipp-20140114/ippScripts/scripts/dist_bundle.pl

@@ -50 +50 @@
                      'PPSUB.INVERSE.MASK' => 'inv_mask',
                      'PPSUB.INVERSE.VARIANCE' => 'inv_variance' );
-my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
+
+my %stack_cleaned = ( 'PPSTACK.UNCONV' => 'image',
+                      'PPSTACK.UNCONV.MASK' => 'mask',
+                      'PPSTACK.UNCONV.VARIANCE' => 'variance',
+                      'PPSTACK.UNCONV.EXP' => 'exp',
+                      'PPSTACK.UNCONV.EXPNUM' => 'expnum',
+                      'PPSTACK.UNCONV.EXPWT' => 'expwt'
+                      );
+
+my %stack_convolved = ( 'PPSTACK.OUTPUT' => 'image',
                       'PPSTACK.OUTPUT.MASK' => 'mask',
-                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
+                      'PPSTACK.OUTPUT.VARIANCE' => 'variance',
+                      'PPSTACK.OUTPUT.EXP' => 'exp',
+                      'PPSTACK.OUTPUT.EXPNUM' => 'expnum',
+                      'PPSTACK.OUTPUT.EXPWT' => 'expwt'
+                      );
 my %empty_cleaned = ();
+
 
 
@@ -136 +150 @@
 if (($stage ne 'raw') and ($stage ne 'fake') and ($stage ne 'stack_summary') and !$poor_quality) {
     # If the file list is empty it is an error because we should at least get a config dump file
-    # except for fake stage which doesn't do anything yet
+    # except for fake stage which doesn't do anything yet, stack_summary stage, and of course raw files have
+    # no config dump because they aren't processed
     &my_die("empty file list", $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
 }
@@ -165 +180 @@
     my $image_type = get_image_type($stage, $file_rule);
 
+    # skip useless empty trace files
+    next if ($file_rule =~ /TRACE/);
+
     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
     next if $image_type and ($clean or $poor_quality);
+
+    if ($stage eq 'stack') {
+        # skip convolved stacks since they are deleted just after they are created now
+        next if $stack_convolved{$file_rule};
+    }
 
     # if magic is required, don't ship jpegs or binned fits images
@@ -180 +203 @@
 
     my $file_name = $file->{name};
-    my $base = basename($file_name);
     my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path and $file_rule =~ /LOG/) {
+        my $compressed_file_name = $file_name . '.bz2';
+        my $compressed_path = $ipprc->file_resolve($compressed_file_name);
+        if ($compressed_path) {
+            $path = $compressed_path;
+            $file_name = $compressed_file_name;
+        }
+    }
+
 
     if (!$path) {
@@ -194 +226 @@
         # XXX: perhaps only do this for stages where we know that this happens
         next if $file_rule =~ /STATS/;
-        # don't fail on these "non-essential files"
+        # don't fail if these "non-essential files" are missing
+        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /TRACE/;
-        next if $file_rule =~ /LOG/;
         next if $file_rule =~ /BIN/;
 
@@ -236 +268 @@
         # create a symbolic link from the file in the nebulous repository
         # in the temporary directory
-        symlink $path, "$tmpdir/$base";
+        symlink $path, "$tmpdir/" . basename($file_name);
     }
 }
@@ -464 +496 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') {
+    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary' or $stage eq 'ff') {
         $type = $empty_cleaned{$rule};
     } else {
@@ -556 +588 @@
         # stack_summary stage does not use a config dump file.
         return build_stack_summary_file_list($path_base, \@file_list);
+    } elsif ($stage eq "ff") {
+        if ($component eq 'summary') {
+            # full force summary does not use a config dump file. Cobble together a file list
+            return build_ff_summary_file_list($path_base, \@file_list);
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -710 +749 @@
 }
 
+# psphotFullForceSummary does not produce a config dump file cobble together a file list
+sub build_ff_summary_file_list {
+    my ($path_base, $file_list) = @_;
+
+    my %cmf ;
+    $cmf{file_rule} = "SOURCES";
+    $cmf{name} = "$path_base.cmf";
+    push @$file_list, \%cmf;
+
+    my %log;
+    $log{file_rule} = "LOG";
+    $log{name} = "$path_base.log";
+    push @$file_list, \%log;
+
+    return $file_list;
+}
+
 sub my_die
 {

tags/ipp-20140114/ippScripts/scripts/queuestaticsky.pl

@@ -33 +33 @@
 };
 
+# don't update the database if we are pretending
+$no_update = 1 if $pretend;
 
 my $missing_tools;
@@ -113 +115 @@
         }
     }
+    last if $pretend;   # only do the first (full) set if we are in pretend mode
 }
 {

tags/ipp-20140114/ippTools/share/Makefile.am

@@ -189 +189 @@
         disttool_definebyquery_diff.sql \
         disttool_definebyquery_fake.sql \
+        disttool_definebyquery_ff.sql \
         disttool_definebyquery_raw.sql \
         disttool_definebyquery_raw_no_magic.sql \
@@ -206 +207 @@
         disttool_pending_diff.sql \
         disttool_pending_fake.sql \
+        disttool_pending_ff.sql \
         disttool_pending_raw.sql \
         disttool_pending_sky.sql \
@@ -496 +498 @@
         fftool_summary.sql \
         fftool_todo.sql \
-        fftool_toadvance.sql
+        fftool_toadvance.sql \
+        fftool_export_run.sql \
+        fftool_export_input.sql \
+        fftool_export_result.sql \
+        fftool_export_summary.sql
 
+

tags/ipp-20140114/ippTools/share/disttool_toadvance.sql

@@ -308 +308 @@
         COUNT(diffSkyfile.skycell_id) = COUNT(distComponent.component)
         AND SUM(distComponent.fault) = 0
+UNION
+-- ff stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    distRun.label,
+    clean
+    -- ,COUNT(warp_id)
+    -- ,COUNT(ff_id)
+    FROM distRun
+    JOIN fullForceRun ON stage_id = ff_id
+    JOIN fullForceResult USING(ff_id)
+    JOIN skycalRun USING(skycal_id)
+    JOIN stackRun USING(stack_id)
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+        AND distComponent.component = CONCAT_WS('.', warp_id, stackRun.skycell_id) 
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'ff'
+    GROUP BY
+        distRun.dist_id,
+        ff_id
+    HAVING
+        -- number of dist components is the number of warps plus 1 (for the summary component)
+        COUNT(fullForceResult.ff_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
 ) as Foo

tags/ipp-20140114/ippTools/src/disttool.c

@@ -413 +413 @@
 
       magic = false;
+    } else if (!strcmp(stage, "ff")) {
+      runJoinStr = "fullForceRun.ff_id";
+      query = pxDataGet("disttool_definebyquery_ff.sql");
+      if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+        psFree(where);
+        return(false);
+      }
+
+      if (label) {
+        psStringAppend(&query, " AND (fullForceRun.label = '%s') ", label);
+      }
+      if (dist_group) {
+        psStringAppend(&query, " AND (fullForceRun.dist_group = '%s') ", dist_group);
+      }
+
+      magic = false;
     } else {
         psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage);

tags/ipp-20140114/ippTools/src/fftool.c

@@ -44 +44 @@
 static bool updatesummaryMode(pxConfig *config);
 static bool summaryMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
+static bool summaryMode(pxConfig *config);
 
 static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state);
@@ -77 +80 @@
         MODECASE(FFTOOL_MODE_UPDATESUMMARY,     updatesummaryMode);
         MODECASE(FFTOOL_MODE_SUMMARY,           summaryMode);
+        MODECASE(FFTOOL_MODE_EXPORTRUN,         exportrunMode);
+        MODECASE(FFTOOL_MODE_IMPORTRUN,         importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -134 +139 @@
 
     psMetadata *warpWhereMD = psMetadataAlloc();
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label",     "warpRun.label",           "LIKE");
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group",      "LIKE");
-    PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",        "warpRun.warp_id",         "==");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label",     "fullForceRun.label",           "LIKE");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","fullForceRun.data_group",      "LIKE");
+    PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id",        "fullForceRun.warp_id",         "==");
     if (!psListLength(warpWhereMD->list)) {
         psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required");
@@ -150 +155 @@
     }
 
-    PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",       "warpRun.tess_id",         "==");
-    pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",        "warpRun.filter",          "LIKE");
+    PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id",       "fullForceRun.tess_id",         "==");
+    pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter",        "fullForceRun.filter",          "LIKE");
     if (!pxskycellAddWhere(config, warpWhereMD)) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
@@ -504 +509 @@
     PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter",      "stackRun.filter", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
     pxskycellAddWhere(config, where);
@@ -864 +870 @@
     PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter",     "stackRun.filter", "LIKE");
     PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
     pxskycellAddWhere(config, where);
@@ -933 +940 @@
     return true;
 }
+
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    // PXOPT_LOOKUP_S64(det_id,  config->args, "-warp_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+//    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
+
+    FILE *f = fopen (outfile, "w");
+    if (f == NULL) {
+        psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+        return false;
+    }
+
+    if (!pxExportVersion(config, f)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+        return false;
+    }
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "==");
+
+    ExportTable tables [] = {
+      {"fullForceRun", "fftool_export_run.sql"},
+      {"fullForceInput", "fftool_export_input.sql"},
+      {"fullForceResult", "fftool_export_result.sql"},
+      {"fullForceSummary", "fftool_export_summary.sql"},
+    };
+
+    int numTables = sizeof(tables)/sizeof(tables[0]);
+
+    for (int i=0; i < numTables; i++) {
+      psString query = pxDataGet(tables[i].sqlFilename);
+      if (!query) {
+          psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+          return false;
+      }
+
+      if (where && psListLength(where->list)) {
+          psString whereClause = psDBGenerateWhereSQL(where, NULL);
+          psStringAppend(&query, " %s", whereClause);
+          psFree(whereClause);
+      }
+
+      // treat limit == 0 as "no limit"
+      if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+      }
+
+      if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+      }
+      psFree(query);
+
+      psArray *output = p_psDBFetchResult(config->dbh);
+      if (!output) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          return false;
+      }
+      if (!psArrayLength(output)) {
+        psError(PS_ERR_UNKNOWN, true, "no rows found");
+        psFree(output);
+        return false;
+      }
+
+      // we must write the export table in non-simple (true) format
+      if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+      }
+      psFree(output);
+    }
+
+    fclose (f);
+
+    return true;
+}
+
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+
+  int numImportTables = 3;
+
+  char tables[3] [80] = {"fullForceInput", "fullForceResult", "fullForceSummary"};
+
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+
+#ifdef notdef
+  fprintf (stderr, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+#endif
+
+  if (!pxCheckImportVersion(config, input)) {
+      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+      return false;
+  }
+
+  psMetadataItem *item = psMetadataLookup (input, "fullForceRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  fullForceRunRow *fullForceRun = fullForceRunObjectFromMetadata (entry->data.md);
+  fullForceRunInsertObject (config->dbh, fullForceRun);
+
+  // fprintf (stdout, "---- warp run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceInputRow *fullForceInput = fullForceInputObjectFromMetadata (entry->data.md);
+          fullForceInputInsertObject (config->dbh, fullForceInput);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceResultRow *fullForceResult = fullForceResultObjectFromMetadata (entry->data.md);
+          fullForceResultInsertObject (config->dbh, fullForceResult);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+
+      case 2:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          fullForceSummaryRow *fullForceSummary = fullForceSummaryObjectFromMetadata (entry->data.md);
+          fullForceSummaryInsertObject (config->dbh, fullForceSummary);
+
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}

tags/ipp-20140114/ippTools/src/fftool.h

@@ -36 +36 @@
     FFTOOL_MODE_REVERTSUMMARY,
     FFTOOL_MODE_UPDATESUMMARY,
-    FFTOOL_MODE_SUMMARY
+    FFTOOL_MODE_SUMMARY,
+    FFTOOL_MODE_EXPORTRUN,
+    FFTOOL_MODE_IMPORTRUN,
 } fftoolMode;
 

tags/ipp-20140114/ippTools/src/fftoolConfig.c

@@ -186 +186 @@
     pxskycellAddArguments(revertsummaryArgs);
 
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0,          "export this full force ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+//    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "export run in cleaned state", false);
+
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
+
+
+
     psFree(now);
 
@@ -203 +216 @@
     PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY,  revertsummaryArgs);
     PXOPT_ADD_MODE("-summary",       "list summary results", FFTOOL_MODE_SUMMARY,  summaryArgs);
+    PXOPT_ADD_MODE("-exportrun",     "list summary results", FFTOOL_MODE_EXPORTRUN,  exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",     "list summary results", FFTOOL_MODE_IMPORTRUN,  importrunArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {