Changeset 33120

Timestamp:

Jan 20, 2012, 9:47:49 AM (14 years ago)

Author:

bills

Message:

Add two columns to pstampJob to enable the parent / child job concept

Location:

tags/ipp-20111222

Files:

• dbconfig/pstamp.md (modified) (1 diff, 1 prop)
• ippTools/share/pstamptool_pendingjob.sql (modified) (2 diffs, 1 prop)
• ippTools/share/pstamptool_stopdependentjob.sql (modified) (1 diff, 1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (7 diffs, 1 prop)
• ippTools/src/pstamptool.c (modified) (8 diffs, 1 prop)
• ippTools/src/pstamptoolConfig.c (modified) (2 diffs, 1 prop)

tags/ipp-20111222/dbconfig/pstamp.md

@@ -48 +48 @@
     dep_id      S64         0
     fault_count S32         0
+    parent_id   S64         0
+    is_parent   BOOL        f
 END
 

tags/ipp-20111222/ippTools/share/pstamptool_pendingjob.sql

@@ -1 @@
-SELECT pstampJob.*,
-    IFNULL(Label.priority, 10000) AS priority
+SELECT * FROM
+(SELECT pstampJob.*,
+    IFNULL(Label.priority, 10000) AS priority,
+    0 AS myParent,
+    0 AS runningChildren
 FROM pstampJob
     JOIN pstampRequest USING(req_id)
@@ -9 +12 @@
     AND pstampJob.state = 'run'
     AND pstampJob.fault = 0
+    AND NOT is_parent
     AND (dep_id = 0 OR pstampDependent.state = 'full')
     AND (Label.active OR Label.active IS NULL)
+    -- where hook 1 %s
+UNION
+SELECT * FROM
+  (SELECT pstampJob.*,
+        IFNULL(Label.priority, 10000) AS priority
+    FROM pstampJob
+        JOIN pstampRequest USING(req_id)
+        LEFT JOIN pstampDependent USING(dep_id)
+        LEFT JOIN Label ON pstampRequest.label = Label.label
+    WHERE pstampRequest.state = 'run'
+        AND pstampRequest.fault = 0
+        AND pstampJob.state = 'run'
+        AND pstampJob.fault = 0
+        AND is_parent
+        AND (dep_id = 0 OR pstampDependent.state = 'full')
+        AND (Label.active OR Label.active IS NULL)
+        -- where hook 2 %s
+    ) as parentJob
+    LEFT JOIN
+    (SELECT parent_id AS myParent,
+            count(job_id) as runningChildren
+        FROM pstampRequest JOIN pstampJob USING(req_id)
+        WHERE pstampRequest.state ='run'
+            AND pstampJob.state = 'run'
+            AND pstampJob.parent_id
+        GROUP BY parent_id
+    ) as childJobs
+    ON childJobs.myParent = parentJob.job_id
+    WHERE runningChildren IS NULL
+) AS theQuery
+
+

tags/ipp-20111222/ippTools/share/pstamptool_stopdependentjob.sql

@@ -5 +5 @@
 SET pstampJob.state = 'stop', 
     pstampJob.fault = %d,
-    pstampDependent.fault = %d 
+    pstampDependent.fault = %d, 
+    pstampDependent.state = '%s'
 WHERE pstampJob.state = 'run'
     AND pstampDependent.state = 'new'

tags/ipp-20111222/ippTools/share/pxadmin_create_tables.sql

@@ -1442 +1442 @@
         fault SMALLINT,
         PRIMARY KEY(req_id),
-        KEY(req_id)
+        KEY(req_id),
+        KEY(state),
+        KEY(fault),
+        KEY(label)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1456 +1459 @@
         options BIGINT,
         dep_id BIGINT,
+        fault_count INT,
+        parent_id BIGINT DEFAULT NULL,
+        is_parent TINYINT DEFAULT 0,
         PRIMARY KEY(job_id, req_id),
         KEY(job_id),
+        KEY(req_id),
+        KEY(state),
+        KEY(fault),
+        KEY(dep_id),
+        KEY(parent_id),
+        KEY(is_parent),
         FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1468 +1480 @@
         component  VARCHAR(64),
         imagedb    VARCHAR(64),
+        outdir     VARCHAR(255),
         rlabel     VARCHAR(64),
         need_magic TINYINT,
-        outdir     VARCHAR(255),
         fault      SMALLINT,
         fault_count INT,
@@ -1477 +1489 @@
         KEY(stage),
         KEY(stage_id),
+        KEY(component),
+        KEY(imagedb),
         KEY(fault)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1964 +1978 @@
        minidvodb_group VARCHAR(64) NOT NULL,
        minidvodb_path VARCHAR(255) NOT NULL,
-       mergedvodb_path VARCHAR(255) NOT NULL,
        state VARCHAR(64) NOT NULL,
        creation_date TIMESTAMP,
@@ -1972 +1985 @@
 CREATE TABLE minidvodbProcessed (
       minidvodb_id BIGINT,
-      merge_order BIGINT,
       dtime_resort FLOAT,
       dtime_relphot FLOAT,
-      dtime_verify FLOAT,
-      dtime_merge FLOAT,
       dtime_script FLOAT,
       epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-      mergedvodb_path VARCHAR(255) NOT NULL,
       fault SMALLINT NOT NULL,
       UNIQUE KEY (minidvodb_id),
@@ -2000 +2009 @@
     FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbRun (
+       merge_id BIGINT AUTO_INCREMENT,
+       minidvodb_id BIGINT,
+       mergedvodb VARCHAR(64) NOT NULL,
+       mergedvodb_path VARCHAR(255) NOT NULL,
+       state VARCHAR(64) NOT NULL,
+       creation_date TIMESTAMP,
+       PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbProcessed (
+      merge_id BIGINT,
+      merge_order BIGINT,
+      dtime_verify FLOAT,
+      dtime_merge FLOAT,
+      dtime_script FLOAT,
+      epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+      fault SMALLINT NOT NULL,
+      UNIQUE KEY (merge_id),
+      KEY(merge_id),
+      FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbCopy (
+    mergedvodbcopy_id     BIGINT AUTO_INCREMENT,
+    merge_id     BIGINT,
+    mergedvodb_rsync_path VARCHAR(255) NOT NULL,
+    destination_host VARCHAR(255) NOT NULL,
+    fault  SMALLINT NOT NULL,
+    state   VARCHAR(64) NOT NULL,
+    epoch TIMESTAMP,
+    dtime FLOAT,
+    PRIMARY KEY(mergedvodbcopy_id),
+    KEY(merge_id),
+    FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE vpRun (
     vp_id BIGINT AUTO_INCREMENT,

tags/ipp-20111222/ippTools/src/pstamptool.c

@@ -704 +704 @@
     PXOPT_LOOKUP_S16(fault,       config->args, "-fault",      false, false);
     PXOPT_LOOKUP_S64(exp_id,      config->args, "-exp_id",     false, false);
-    PXOPT_LOOKUP_S64(options,     config->args, "-options",     false, false);
+    PXOPT_LOOKUP_S64(options,     config->args, "-options",    false, false);
     PXOPT_LOOKUP_S64(dep_id,      config->args, "-dep_id",     false, false);
+    PXOPT_LOOKUP_S64(parent_id,   config->args, "-parent_id",  false, false);
+    PXOPT_LOOKUP_BOOL(is_parent,  config->args, "-is_parent",  false);
 
     // unless the job is being inserted with stop state require outputBase
@@ -716 +718 @@
     if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) {
         stampJob = false;
+    } else if (!strcmp(job_type, "child")) {
+        // job_type child's only action is to resolve a dependent
+        // XXX: IS this necessary?
+        if (!dep_id) {
+            psError(PS_ERR_UNKNOWN, true, "dep_id required for child job\n");
+            return false;
+        }
+        if (is_parent) {
+            psError(PS_ERR_UNKNOWN, true, "job type child can not be a parent job\n");
+            return false;
+        }
+        stampJob = false;
+    } else if (!strcmp(job_type, "mosaic")) {
+        stampJob = false;
+        is_parent = true;
     } else if (!strcmp(job_type, "stamp")) {
         stampJob = true;
@@ -723 +740 @@
     }
     if (!pstampJobInsert(config->dbh,
-            0, // job_id
+            0,          // job_id
             req_id,
             rownum,
@@ -733 +750 @@
             options,
             dep_id,
-            0   // fault_count
+            0,          // fault_count
+            parent_id,
+            is_parent
             )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -842 +861 @@
 
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    psString whereStr = NULL;
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&whereStr, "\n AND %s", whereClause);
         psFree(whereClause);
+    } else {
+        psStringAppend(&whereStr, "%s", "");
     }
     psFree(where);
@@ -858 +880 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
+    if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereStr);
+        psFree(query);
+        return false;
+    }
+    psFree(whereStr);
     psFree(query);
 
@@ -956 +980 @@
 
     PXOPT_LOOKUP_S32(fault,  config->args, "-set_fault",  true, false);
+    PXOPT_LOOKUP_STR(state,  config->args, "-set_state",  false, false);
 
     psMetadata *where = psMetadataAlloc();
@@ -986 +1011 @@
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) {
+    if (!p_psDBRunQueryF(config->dbh, query, fault, fault, state)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);

tags/ipp-20111222/ippTools/src/pstamptoolConfig.c

@@ -144 +144 @@
     psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0,          "define options", 0);
     psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0,           "define job dep_id", 0);
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-parent_id", 0,        "define parent's job_id", 0);
+    psMetadataAddBool(addjobArgs, PS_LIST_TAIL, "-is_parent", 0,       "define whether job has children", false);
     psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0,            "define job result", 0);
 
@@ -186 +188 @@
     psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL,  "-fault_count", 0,   "select by fault_count (>=)", 0);
     psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-set_fault", 0,    "new fault value for job and dependent (required)", 0);
+    psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-set_state", 0,            "new pstampDependent.state", "new");
     psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
     psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0,      "not used", 0);