Changeset 32337 for branches/eam_branches/ipp-20110710/ippTools

Timestamp:

Sep 6, 2011, 11:00:22 AM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20110710/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/camtool_find_pendingimfile.sql (modified) (1 prop)
• share/dettool_toresidimfile.sql (modified) (1 diff)
• share/difftool_pendingcleanuprun.sql (modified) (5 diffs)
• share/dqstatstool_definebyquery.sql (modified) (1 diff)
• share/magictool_definebyquery_select_multidiff.sql (copied) (copied from trunk/ippTools/share/magictool_definebyquery_select_multidiff.sql )
• share/pubtool_definerun.sql (modified) (2 diffs)
• share/pxadmin_create_tables.sql (modified) (4 diffs, 1 prop)
• src (modified) (1 prop)
• src/bgtoolConfig.c (modified) (2 diffs)
• src/chiptool.c (modified) (2 diffs)
• src/chiptoolConfig.c (modified) (1 diff)
• src/difftool.c (modified) (1 diff)
• src/disttool.c (modified) (1 diff)
• src/disttoolConfig.c (modified) (1 diff)
• src/dqstatstool.c (modified) (1 diff)
• src/magictool.c (modified) (2 diffs, 1 prop)
• src/magictoolConfig.c (modified) (1 diff)
• src/regtool.c (modified) (2 diffs)
• src/regtoolConfig.c (modified) (1 diff)
• src/stacktool.c (modified) (2 diffs)
• src/warptool.c (modified) (6 diffs)
• src/warptoolConfig.c (modified) (1 diff)

branches/eam_branches/ipp-20110710/ippTools/share/Makefile.am

@@ -244 +244 @@
         magictool_definebyquery_insert.sql \
         magictool_definebyquery_select.sql \
+        magictool_definebyquery_select_multidiff.sql \
         magictool_inputs.sql \
         magictool_inputskyfile.sql \

branches/eam_branches/ipp-20110710/ippTools/share/dettool_toresidimfile.sql

@@ -101 +101 @@
  AND detResidImfile.exp_id IS NULL
  AND detResidImfile.class_id IS NULL
+UNION
+SELECT
+    detRun.det_id,
+    detRun.iteration,
+    detRun.det_type,
+    detRun.mode,
+    detRun.workdir,
+    detRun.reduction,
+    detProcessedImfile.exp_id,
+    detProcessedImfile.class_id,
+    detProcessedImfile.uri,
+    detRegisteredImfile.uri AS det_uri,
+    detRun.ref_det_id,
+    detRun.ref_iter,
+    rawExp.camera,
+    rawExp.exp_tag
+FROM detRun
+JOIN detRun AS detRunRef
+  ON detRunRef.det_id = detRun.ref_det_id
+JOIN detInputExp
+  ON detInputExp.det_id    = detRun.det_id
+ AND detInputExp.iteration = detRun.iteration
+JOIN rawExp
+  ON rawExp.exp_id = detInputExp.exp_id
+JOIN detProcessedImfile
+  ON detProcessedImfile.det_id    = detRun.det_id
+ AND detProcessedImfile.exp_id    = detInputExp.exp_id
+JOIN detRegisteredImfile
+  ON detRegisteredImfile.det_id    = detRun.ref_det_id
+ AND detRegisteredImfile.iteration = detRun.ref_iter
+ AND detRegisteredImfile.class_id  = detProcessedImfile.class_id
+LEFT JOIN detResidImfile
+  ON detResidImfile.det_id    = detRun.det_id      
+ AND detResidImfile.iteration = detRun.iteration   
+ AND detResidImfile.exp_id    = detProcessedImfile.exp_id   
+ AND detResidImfile.class_id  = detProcessedImfile.class_id 
+WHERE
+ detRun.state = 'run'
+ AND detRun.mode = 'verify'
+ AND detRunRef.state = 'stop'
+ AND detRegisteredImfile.fault = 0
+ AND detProcessedImfile.fault = 0
+ AND detResidImfile.det_id IS NULL
+ AND detResidImfile.iteration IS NULL
+ AND detResidImfile.exp_id IS NULL
+ AND detResidImfile.class_id IS NULL
 ORDER BY exp_id

branches/eam_branches/ipp-20110710/ippTools/share/difftool_pendingcleanuprun.sql

@@ -2 +2 @@
 -- all of this is just to get the camera used for the diff run
 SELECT DISTINCT * FROM 
-   (SELECT 
+   (SELECT DISTINCT
             diffRun.diff_id,
             rawExp.camera,
@@ -29 +29 @@
              diffRun.state = 'goto_scrubbed' OR 
              diffRun.state = 'goto_purged')
+        @INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT
             diffRun.diff_id,
             rawExp.camera,
@@ -57 +58 @@
              diffRun.state = 'goto_scrubbed' OR 
              diffRun.state = 'goto_purged')
+        @INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT 
             diffRun.diff_id,
             rawExp.camera,
@@ -85 +87 @@
              diffRun.state = 'goto_scrubbed' OR 
              diffRun.state = 'goto_purged')
+        @INNERLIMITS@
      UNION
-     SELECT
+     SELECT DISTINCT
             diffRun.diff_id,
             rawExp.camera,
@@ -113 +116 @@
              diffRun.state = 'goto_scrubbed' OR 
              diffRun.state = 'goto_purged')
+        @INNERLIMITS@
         ) as Foo
         WHERE 1

branches/eam_branches/ipp-20110710/ippTools/share/dqstatstool_definebyquery.sql

@@ -1 +1 @@
 -- query to identify unique data to bundle
 
-SELECT DISTINCT exp_id,chip_id,cam_id,warp_id,invalid from rawExp 
+SELECT DISTINCT exp_id,chip_id,cam_id,warpRun.warp_id,invalid from rawExp 
         JOIN chipRun USING(exp_id) 
         JOIN camRun USING(chip_id) 
+        JOIN camProcessedExp USING(cam_id)
         LEFT OUTER JOIN fakeRun USING(cam_id) 
         LEFT OUTER JOIN warpRun USING(fake_id)
-        LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id,warp_id)
+        LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id)
 WHERE camRun.state = 'full'

branches/eam_branches/ipp-20110710/ippTools/share/pubtool_definerun.sql

@@ -1 @@
--- Get runs to publish
+-- Get runs to publish 
 SELECT DISTINCT
     client_id,
@@ -20 +20 @@
     JOIN chipRun USING(chip_id)
     JOIN rawExp USING(exp_id)
+    LEFT JOIN magicDSRun ON magicDSRun.stage = 'diff' AND magicDSRun.stage_id = diff_id
     WHERE publishClient.stage = 'diff'
         AND publishClient.active = 1
         AND diffRun.state IN ('full', 'cleaned', 'goto_cleaned')
-        AND (diffRun.magicked != 0 OR diffRun.diff_mode = 4 OR publishClient.magicked = 0)
+        AND ( diffRun.diff_mode = 4
+           OR (publishClient.magicked AND diffRun.magicked != 0)
+           OR (publishClient.magicked = 0 AND 
+                (diffRun.magicked = 0 OR (diffRun.magicked != 0 AND magic_ds_id IS NOT NULL))
+              )
+           )
     -- WHERE hook %s
     UNION

branches/eam_branches/ipp-20110710/ippTools/share/pxadmin_create_tables.sql

@@ -533 +533 @@
 CREATE TABLE addRun (
     add_id BIGINT AUTO_INCREMENT,
-    cam_id BIGINT,
+    stage VARCHAR(64),
+    stage_id BIGINT,
     state VARCHAR(64),
     workdir VARCHAR(255),
@@ -543 +544 @@
     note  VARCHAR(255),
     image_only TINYINT,
+    minidvodb TINYINT, 
+    minidvodb_group VARCHAR(64),
+    minidvodb_name VARCHAR(64),
     PRIMARY KEY(add_id),
     KEY(add_id),
-    KEY(cam_id),
+    KEY(stage_id),
+    KEY(stage),
     KEY(state),
     KEY(workdir_state),
     KEY(label),
     KEY(data_group),
-    INDEX(add_id, cam_id),
-    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+    INDEX(add_id, cam_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -558 +562 @@
     dtime_addstar FLOAT,
     path_base VARCHAR(255),
+    dvodb_path VARCHAR(255),
     fault SMALLINT NOT NULL,
     PRIMARY KEY(add_id),
@@ -1946 +1951 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;    
 
+CREATE TABLE minidvodbRun (
+minidvodb_id BIGINT AUTO_INCREMENT,
+       minidvodb_name VARCHAR(64),
+       minidvodb_group VARCHAR(64) NOT NULL,
+       minidvodb_path VARCHAR(255) NOT NULL,
+       mergedvodb_path VARCHAR(255) NOT NULL,
+       state VARCHAR(64) NOT NULL,
+       creation_date TIMESTAMP,
+       PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE minidvodbProcessed (
+      minidvodb_id BIGINT,
+      merge_order BIGINT,
+      dtime_resort FLOAT,
+      dtime_relphot FLOAT,
+      dtime_verify FLOAT,
+      dtime_merge FLOAT,
+      dtime_script,
+      epoch TIMESTAMP,
+      mergedvodb_path VARCHAR(255) NOT NULL,
+      fault SMALLINT NOT NULL,
+       KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+CREATE TABLE minidvodbCopy (
+    minidvodbcopy_id     BIGINT AUTO_INCREMENT,
+    minidvodb_id     BIGINT,
+    minidvodb_rsync_path VARCHAR(255) NOT NULL,
+    destination_host VARCHAR(255) NOT NULL,
+    fault  SMALLINT NOT NULL,
+    state   VARCHAR(64) NOT NULL,
+    epoch TIMESTAMP,
+    dtime FLOAT,
+    PRIMARY KEY(minidvodbcopy_id),
+    KEY(minidvodb_id),
+    FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+
+
 -- This comment line is here to avoid empty query error.
 -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.

branches/eam_branches/ipp-20110710/ippTools/src/bgtoolConfig.c

@@ -113 +113 @@
     psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
     psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0); 
+    psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-not_destreaked",  0,        "define magicked status", 0); 
     psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
     psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
@@ -298 +300 @@
     psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
     psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0); 
     psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
     psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);

branches/eam_branches/ipp-20110710/ippTools/src/chiptool.c

@@ -743 +743 @@
         faulted = false;
     }
+    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
 
     psMetadata *where = psMetadataAlloc();
@@ -787 +788 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
+    }
+
+    if (pstamp_order) {
+        // put runs in order of exposure id with newest chip Runs first
+        // The postage stamp parser depends on this behavior
+        psStringAppend(&query, "\nORDER by exp_id, chip_id DESC");
     }
 

branches/eam_branches/ipp-20110710/ippTools/src/chiptoolConfig.c

@@ -210 +210 @@
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL);
     pxmagicAddArguments(processedimfileArgs);
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-pstamp_order",  0,    "order results for postage stamp server", false);
 
     psMetadataAddU64(processedimfileArgs,  PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);

branches/eam_branches/ipp-20110710/ippTools/src/difftool.c

@@ -2461 +2461 @@
         psString limitString = psDBGenerateLimitSQL(limit);
         psStringAppend(&query, " %s", limitString);
+        psStringSubstitute(&query,limitString,"@INNERLIMITS@");
         psFree(limitString);
+
+    }
+    else {
+      psStringSubstitute(&query,NULL,"@INNERLIMITS@");
     }
     //    fprintf(stderr,"%s",query);

branches/eam_branches/ipp-20110710/ippTools/src/disttool.c

@@ -771 +771 @@
     PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "distRun.label", "==");
 
     PXOPT_COPY_S16(config->args, where,  "-fault", "distRun.fault", "==");

branches/eam_branches/ipp-20110710/ippTools/src/disttoolConfig.c

@@ -105 +105 @@
     psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0);
     psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-state",    0, "define state", NULL);
-    psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label",    0, "define label", NULL);
+    psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
     psMetadataAddBool(revertrunArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);

branches/eam_branches/ipp-20110710/ippTools/src/dqstatstool.c

@@ -92 +92 @@
   }
   else {
-    psStringAppend(&query, " AND (warp_id IS NOT NULL AND invalid IS NULL) ");
+    psStringAppend(&query, " AND ((warpRun.warp_id IS NOT NULL OR camProcessedExp.quality > 0) AND invalid IS NULL) ");
   }
 

branches/eam_branches/ipp-20110710/ippTools/src/magictool.c

@@ -131 +131 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
-
+    PXOPT_LOOKUP_BOOL(multidiff, config->args, "-multidiff", false);
     psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
     PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
@@ -142 +142 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select.sql");
+        psString query;
+        if (multidiff) {
+          query = pxDataGet("magictool_definebyquery_select_multidiff.sql");
+        }
+        else {
+          query = pxDataGet("magictool_definebyquery_select.sql");
+        }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

branches/eam_branches/ipp-20110710/ippTools/src/magictoolConfig.c

@@ -61 +61 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-multidiff", 0, "allow multiple diffs to be magicked with the same label", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);

branches/eam_branches/ipp-20110710/ippTools/src/regtool.c

@@ -720 +720 @@
     PXOPT_COPY_STR(config->args, where,  "-obs_mode",      "obs_mode", "==");
     PXOPT_COPY_STR(config->args, where,  "-data_state",    "data_state", "==");
+    PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -769 +770 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
+    }
+    if (video_cells) {
+        psStringAppend(&query, " %s", "AND rawImfile.video_cells");
     }
 

branches/eam_branches/ipp-20110710/ippTools/src/regtoolConfig.c

@@ -169 +169 @@
     ADD_OPT(Bool, processedimfileArgs, "-simple",    "use the simple output format",          false);
     ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false);
+    ADD_OPT(Bool, processedimfileArgs, "-video_cells",    "limit result to chips that have video cells", false);
     pxspaceAddArguments(processedimfileArgs);
     pxmagicAddArguments(processedimfileArgs);

branches/eam_branches/ipp-20110710/ippTools/src/stacktool.c

@@ -154 +154 @@
     psMetadata *row = output->data[i];
 
-    printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"),
-           psMetadataLookupStr(NULL,row,"data_group"),
-           psMetadataLookupStr(NULL,row,"tess_id"),
-           psMetadataLookupStr(NULL,row,"filter"),
-           psMetadataLookupStr(NULL,row,"projection_cell"));
+/*     printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"), */
+/*            psMetadataLookupStr(NULL,row,"data_group"), */
+/*            psMetadataLookupStr(NULL,row,"tess_id"), */
+/*            psMetadataLookupStr(NULL,row,"filter"), */
+/*            psMetadataLookupStr(NULL,row,"projection_cell")); */
 
     if (psMetadataLookupS64(NULL,row,"sass_id") == PS_MAX_S64) {
@@ -172 +172 @@
                      psMetadataLookupStr(NULL,row,"projection_cell"));
   }
-  printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"),
-         psMetadataLookupStr(NULL,outrow,"data_group"),
-         psMetadataLookupStr(NULL,outrow,"tess_id"),
-         psMetadataLookupStr(NULL,outrow,"filter"),
-         psMetadataLookupStr(NULL,outrow,"projection_cell"));
+/*   printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"), */
+/*          psMetadataLookupStr(NULL,outrow,"data_group"), */
+/*          psMetadataLookupStr(NULL,outrow,"tess_id"), */
+/*          psMetadataLookupStr(NULL,outrow,"filter"), */
+/*          psMetadataLookupStr(NULL,outrow,"projection_cell")); */
 
 

branches/eam_branches/ipp-20110710/ippTools/src/warptool.c

@@ -975 +975 @@
     psFree(where);
 
+    psArray *allOutput = psArrayAllocEmpty(0);
     // Now loop over the labels and query each individually
     for (long i = 0; i < psArrayLength(labelOutput); i++) {
@@ -1023 +1024 @@
         long outputLength = psArrayLength(output);
         if (outputLength) {
-            // negative simple so the default is true
-            if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to print array");
-                psFree(output);
-                return false;
+            for (int i = 0; i< outputLength; i++) {
+                psPtr ptr = output->data[i];
+                allOutput = psArrayAdd(allOutput, outputLength - i + 10, ptr);
+                if (!allOutput) {
+                    psError(PS_ERR_UNKNOWN, false, "failed to add element to array");
+                    return false;
+                }
             }
         } else {
@@ -1044 +1047 @@
         }
     }
+    long allLabelsLength = psArrayLength(allOutput);
+    if (allLabelsLength) {
+        if (!ippdbPrintMetadatas(stdout, allOutput, "warpPendingSkyCell", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(allOutput);
+            return false;
+        }
+    }
+    psFree(allOutput);
     psFree(labelOutput);
     psFree(whereStr);
@@ -1382 +1394 @@
     pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
 
+    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
     PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
@@ -1423 +1436 @@
     }
     psFree(where);
+
+    if (pstamp_order) {
+        // put runs in order of exposure id with newest warp Runs first
+        // The postage stamp parser depends on this behavior
+        psStringAppend(&query, "\nORDER by exp_id, warp_id DESC");
+    }
 
     // treat limit == 0 as "no limit"
@@ -2467 +2486 @@
 
     if (pstamp_order) {
-        // put runs in order of exposure id with newest chip Runs first
+        // put runs in order of exposure id with newest warp Runs first
         // The postage stamp parser depends on this behavior
         psStringAppend(&query, "\nORDER by exp_id, warp_id DESC");
     }
-
 
     // treat limit == 0 as "no limit"

branches/eam_branches/ipp-20110710/ippTools/src/warptoolConfig.c

@@ -258 +258 @@
     // add arguments for spatial search
     pxspaceAddArguments(warpedArgs);
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-pstamp_order",  0,    "order results for postage stamp server", false);
 
     psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);