Changeset 35546

Timestamp:

May 9, 2013, 11:26:59 AM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130419

Files:

• Nebulous (modified) (1 prop)
• ippMonitor (modified) (1 prop)
• ippTasks (modified) (1 prop)
• ippTasks/nightly_stacks.pro (modified) (1 diff)
• ippTasks/pstamp.pro (modified) (1 diff)
• ippTasks/survey.pro (modified) (7 diffs)
• ippTools/share (modified) (1 prop)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_pendingcleanupimfile.sql (modified) (1 diff)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (4 diffs, 1 prop)
• ippTools/share/releasetool_definerelstack.sql (modified) (2 diffs)
• ippTools/share/releasetool_definerelstack_with_skycal.sql (modified) (1 diff)
• ippTools/share/releasetool_listrelstack.sql (modified) (1 diff)
• ippTools/share/warptool_pendingcleanupskyfile.sql (modified) (1 diff)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/camtool.c (modified) (1 prop)
• ippTools/src/camtoolConfig.c (modified) (1 prop)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/releasetool.c (modified) (11 diffs)
• ippTools/src/releasetoolConfig.c (modified) (2 diffs)

branches/eam_branches/ipp-20130419/ippTasks/nightly_stacks.pro

@@ -621 +621 @@
   periods         -poll $LOADPOLL
   periods         -exec $LOADEXEC
-  periods         -timeout 120
+  periods         -timeout 900 # this task can be quite slow
   npending        1
 

branches/eam_branches/ipp-20130419/ippTasks/pstamp.pro

@@ -965 +965 @@
 
         # XXX: have the script set this up this
-        $MYLOGFILE=/data/ippdb02.0/pstamp/work/logs/cleanup.$REQ_ID
+        $MYLOGFILE=/data/ippc17.0/pstamp/work/logs/cleanup.$REQ_ID
         stdout $MYLOGFILE
         stderr $MYLOGFILE

branches/eam_branches/ipp-20130419/ippTasks/survey.pro

@@ -20 +20 @@
  book create SURVEY_LAPGROUP
  book create SURVEY_RELEXP
+ book create SURVEY_RELSTACK
  $haveSurveyBooks = TRUE
 end
@@ -37 +38 @@
 $SURVEY_LAPGROUP_DB = 0
 $SURVEY_RELEXP_DB = 0
+$SURVEY_RELSTACK_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
 $SURVEY_EXEC = 120
 $SURVEY_POLL = 10
 $SURVEY_TIMEOUT = 600
+# the tasks for loading the release tabels for nightly science do not need to run very often
+$SURVEY_RELEASE_EXEC 300
 
 macro survey.on
@@ -88 +92 @@
     active true
   end
+  task survey.relstack
+    active true
+  end
 end
 
@@ -134 +141 @@
   end
   task survey.relexp
+    active false
+  end
+  task survey.relstack
     active false
   end
@@ -568 +578 @@
   end
   book listbook SURVEY_RELEXP
+end
+
+macro survey.add.relstack
+  if ($0 != 3)
+    echo "USAGE: survey.add.relstack (label) (releasename)"
+    break
+  end
+  book newpage SURVEY_RELSTACK $1
+  book setword SURVEY_RELSTACK $1 LABEL $1
+  book setword SURVEY_RELSTACK $1 RELEASE_NAME $2
+  book setword SURVEY_RELSTACK $1 STATE PENDING
+end
+
+macro survey.del.relstack
+  if ($0 != 2)
+    echo "USAGE: survey.del.relstack (label)"
+    break
+  end
+  book delpage SURVEY_RELSTACK $1
+end
+
+macro survey.show.relstack
+  if ($0 != 1)
+    echo "USAGE: survey.show.relstack"
+    break
+  end
+  book listbook SURVEY_RELSTACK
 end
 
@@ -1706 +1743 @@
  
   periods      -poll $SURVEY_POLL
-  periods      -exec $SURVEY_EXEC
+  periods      -exec $SURVEY_RELEASE_EXEC
   periods      -timeout $SURVEY_TIMEOUT
   npending     1
@@ -1770 +1807 @@
   end
 end
+
+task survey.relstack
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_RELEASE_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.relstack.log
+  stderr $LOGDIR/survey.relstack.log
+
+  task.exec
+    book npages SURVEY_RELSTACK -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_RELSTACK 0 -var label -key STATE NEW
+    if ("$label" == "NULL")
+      # All labels have been done --- reset
+#      echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_RELSTACK $i -var label
+        book setword SURVEY_RELSTACK $label STATE NEW
+      end
+      book getpage SURVEY_RELSTACK 0 -var label -key STATE NEW
+
+      # Select different database
+      $SURVEY_RELSTACK_DB ++
+      if ($SURVEY_RELSTACK_DB >= $DB:n) set SURVEY_RELSTACK_DB = 0
+    end
+
+    book setword SURVEY_RELSTACK $label STATE DONE
+    book getword SURVEY_RELSTACK $label RELEASE_NAME -var RELEASE_NAME
+
+    # XXX: Can we use the survey task for stack types other than nightly?
+    $run = releasetool -definerelstack -label $label -release_name $RELEASE_NAME -set_state processed -set_stack_type nightly
+
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run -dbname $DB:$SURVEY_RELSTACK_DB
+      option $DB:$SURVEY_RELSTACK_DB
+    end
+    
+     echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20130419/ippTools/share/chiptool_pendingcleanupimfile.sql

@@ -13 +13 @@
     USING(chip_id)
 WHERE
-    ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full'
-                                       OR chipProcessedImfile.data_state = 'update') AND chipProcessedImfile.quality = 0)
+    ((chipRun.state = 'goto_cleaned'
+        AND (chipProcessedImfile.data_state = 'full'
+          OR chipProcessedImfile.data_state = 'update'
+          OR chipProcessedImfile.data_state = 'error_cleaned')
+        AND chipProcessedImfile.quality = 0)
 OR 
     (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed')

branches/eam_branches/ipp-20130419/ippTools/share/pxadmin_create_tables.sql

@@ -1 @@
-CREATE TABLE dbversion (
+eREATE TABLE dbversion (
     schema_version VARCHAR(64),
     updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -2172 +2172 @@
     priority    INT,
     PRIMARY KEY(rel_id),
+    UNIQUE KEY(surveyID, release_name),
     KEY(release_name),
     KEY(release_state),
@@ -2196 +2197 @@
     PRIMARY KEY (relexp_id),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
+    UNIQUE KEY(rel_id, exp_id),
     KEY (state),
     KEY (fault),
@@ -2224 +2226 @@
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
+    UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
     KEY (stack_type),
     KEY (mjd_obs),

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack.sql

@@ -6 +6 @@
     stackRun.tess_id,
     stackRun.filter,
-    0 AS mjd_obs,
+    stackSumSkyfile.mjd_obs,
     0 AS zpt_obs,
     0 AS zpt_stdev
@@ -12 +12 @@
 FROM ippRelease
 JOIN stackRun
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
-
+JOIN stackSumSkyfile using(stack_id)
+LEFT JOIN relStack AS previousRelStack 
+    ON previousRelStack.rel_id = ippRelease.rel_id
+    AND previousRelStack.tess_id = stackRun.tess_id 
+    AND previousRelStack.skycell_id = stackRun.skycell_id
+    AND previousRelStack.filter = stackRun.filter
+    -- JOIN hook %s
 WHERE previousRelStack.relstack_id IS NULL
+    AND stackRun.state ='full'
+    AND stackSumSkyfile.quality = 0
+    AND stackSumSkyfile.fault = 0

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack_with_skycal.sql

@@ -13 +13 @@
 JOIN stackRun
 JOIN stackSumSkyfile USING(stack_id)
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
 JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id
 JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id
+-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
+LEFT JOIN relStack AS previousRelStack 
+    ON ippRelease.rel_id = previousRelStack.rel_id 
+    AND stackRun.tess_id = previousRelStack.tess_id 
+    AND stackRun.skycell_id = previousRelStack.skycell_id 
+    AND stackRun.filter = previousRelStack.filter
+-- JOIN Hook %s
 
 WHERE previousRelStack.relstack_id IS NULL
+    AND skycalRun.state = 'full'

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_listrelstack.sql

@@ -7 +7 @@
     stackRun.state AS stack_state,
     stackRun.data_group AS stack_data_group,
+    skycalResult.fwhm_major,
     skycalResult.path_base AS skycal_path_base,
     skycalRun.data_group AS skycal_data_group,

branches/eam_branches/ipp-20130419/ippTools/share/warptool_pendingcleanupskyfile.sql

@@ -10 +10 @@
     USING(warp_id)
 WHERE
-   ((warpRun.state = 'goto_cleaned'  AND (warpSkyfile.data_state = 'full'
-                                      OR  warpSkyfile.data_state = 'update')
-    AND warpSkyfile.quality = 0)
+   ((warpRun.state = 'goto_cleaned'  
+       AND (warpSkyfile.data_state = 'full'
+          OR  warpSkyfile.data_state = 'error_cleaned'
+          OR  warpSkyfile.data_state = 'update')
+       AND warpSkyfile.quality = 0)
     OR
     (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed')

branches/eam_branches/ipp-20130419/ippTools/src/releasetool.c

@@ -702 +702 @@
     PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",      "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",   "warpSkyfile.skycell_id", "==");
+    
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false);
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
@@ -715 +719 @@
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
+    }
+
+    if (skycell_id) {
+        psStringAppend(&query, "\nJOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id AND warpRun.tess_id = warpSkyfile.tess_id");
     }
 
@@ -741 +749 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC");
     }
 
@@ -787 +795 @@
     bool includeSkycal = false;
 
+    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "==");
     if (psListLength(where->list)) {
         includeSkycal = true;
@@ -795 +803 @@
 
     PXOPT_COPY_STR(config->args, where, "-label",      "stackRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-stack_id",   "stackRun.stack_id", "==");
 
@@ -848 +856 @@
     }
 
+    psString joinConditions = psStringCopy("");
+    // Finish up the selectors for the JOIN to previousRelStack
+    // join to rows with our stack_type
+    psStringAppend(&joinConditions, "\nAND previousRelStack.stack_type = '%s'", stack_type);
+
+    if (!strcmp(stack_type, "nightly")) {
+        // detect nightly stack entries that already exist for this day
+        psStringAppend(&joinConditions, "\nAND previousRelStack.mjd_obs = floor(stackSumSkyfile.mjd_obs)");
+    }
+
+    // Add in the where conditions
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-    psStringAppend(&query, " AND %s", whereClause);
+    psStringAppend(&query, "\nAND %s", whereClause);
     psFree(whereClause);
+
+    if (!strcmp(stack_type, "nightly")) {
+        // avoid stacks with NAN mjd_obs. These are old skycells that were lost due to system failure
+        // prior to the time that the mjd_obs was extracted from the headers and added to stackSumSkyfile
+        psStringAppend(&query, "\nAND stackSumSkyfile.mjd_obs IS NOT NULL");
+    }
 
     psFree(where);
@@ -860 +885 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, joinConditions)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        psFree(joinConditions);
         return false;
     }
     psFree(query);
+    psFree(joinConditions);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -985 +1012 @@
     PXOPT_COPY_F32(config->args, where, "-mjd_min",    "stackSumSkyfile.mjd_obs", ">=");
     PXOPT_COPY_F32(config->args, where, "-mjd_max",    "stackSumSkyfile.mjd_obs", "<=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_minor", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
     PXOPT_COPY_STR(config->args, where, "-tess_id",     "relStack.tess_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id",  "relStack.skycell_id", "LIKE");
+//    PXOPT_COPY_STR(config->args, where, "-stack_type",  "relStack.stack_type", "==");
     PXOPT_COPY_STR(config->args, where, "-stack_data_group",  "stackRun.data_group", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
@@ -996 +1022 @@
     pxskycellAddWhere(config, where);
 
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "IFNULL(skycalResult.fwhm_major, 999)", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "IFNULL(skycalResult.fwhm_major, 0)", "<=");
+
     PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "==");
 
     psString query = pxDataGet("releasetool_listrelstack.sql");
@@ -1018 +1049 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC");
     }
 
@@ -1487 +1518 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
     PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalResult.fwhm_major", "<=");
     PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");

branches/eam_branches/ipp-20130419/ippTools/src/releasetoolConfig.c

@@ -164 +164 @@
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL);
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-tess_id", 0,    "warpRun.tess_id", NULL);
 
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
@@ -220 +222 @@
     psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN);
 
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_min", 0, "search by MJD seeing (>=)", NAN);
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_max", 0, "search by MJD (<=)", NAN);
+
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL);
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL);
 
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);