Index: branches/czw_branch/20101203/ippTools/src/disttool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/disttool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/disttool.c (revision 30586) @@ -857,4 +857,6 @@ psFree(where); + psStringAppend(&query, "\nORDER BY priority DESC, dist_id"); + // treat limit == 0 as "no limit" if (limit) { Index: branches/czw_branch/20101203/ippTools/src/dqstatstool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/dqstatstool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/dqstatstool.c (revision 30586) @@ -92,5 +92,5 @@ } else { - psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); + psStringAppend(&query, " AND (warp_id IS NOT NULL AND invalid IS NULL) "); } @@ -402,7 +402,17 @@ if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& - isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& - isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"STRICT") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM")))) { psArrayAdd(decisionRules,decisionRules->n,enginedef); + } + else if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& + (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"CDF") == 0)&& + (isfinite(psMetadataLookupF32(&status,enginedef,"CDF00"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF50"))&& + isfinite(psMetadataLookupF32(&status,enginedef,"CDF100")))) { + psArrayAdd(decisionRules,decisionRules->n, enginedef); + } + else if (strcmp(psMetadataLookupStr(&status,enginedef,"RULETYPE"),"UNUSED") == 0) { } else { @@ -637,4 +647,5 @@ // Calculate if this exposure was "good" or not. bool accept = true; + psF32 quality = 1.0; for (long j = 0; j < decisionRules->n; j++) { psMetadata *rule = decisionRules->data[j]; @@ -644,29 +655,10 @@ psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); return(false); - } - psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); - return(false); - } - psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); - if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); - return(false); - } - psString filter = psMetadataLookupStr(&status,rule,"FILTER"); - if (!status) { - status = true; - } - if (filter) { - psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); - if (strcmp(filter,imfilter) != 0) { - continue; - } } // Not happy with this being set to a F32. Can this ever be something else? psF32 value = psMetadataLookupF32(&status,tableRow,colname); if (!status) { - psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f) in compare with rule.", + colname,value); psFree(colNames); psFree(colTypes); @@ -677,7 +669,92 @@ return(false); } - if ((value > max_value)||(value < min_value)) { - accept = false; - } + psString filter = psMetadataLookupStr(&status,rule,"FILTER"); + if (!status) { + status = true; + } + if (filter) { + psString imfilter = psMetadataLookupStr(&status,tableRow,"FILTER"); + if (strcmp(filter,imfilter) != 0) { + continue; + } + } + + if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"STRICT") == 0) { + psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); + return(false); + } + psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); + if (!status) { + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); + return(false); + } + + if ((value > max_value)||(value < min_value)) { + accept = false; + } + } // End STRICT min/max rule parsing + else if (strcmp(psMetadataLookupStr(&status,rule,"RULETYPE"),"CDF") == 0) { + // I'm just going to implement a quick linear interpolation over the points that exist. + psF32 min_value,max_value = 99; + psF32 min_cdf = 0.0,max_cdf = 1.0; + psF32 cdf; + psF32 try[22] = {0.00,0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50, + 0.55,0.60,0.65,0.70,0.75,0.80,0.85,0.90,0.95,0.99,1.00}; + int i; + int done = 0; + status = true; + min_value = psMetadataLookupF32(&status,rule,"CDF00"); + min_cdf = 0.0; + if (!isfinite(value)) { + quality = 0.0; + continue; + } + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find CDF00 value for %s",colname); + } + if (value < min_value) { + if (quality > min_cdf) { + quality = 0.0; + continue; + } + } + + for (i = 1; i < 22; i++) { + psString label = psStringCopy(""); + psStringAppend(&label,"CDF%02d",(int) (100 * try[i])); + + max_value = psMetadataLookupF32(&status,rule,label); + max_cdf = try[i]; + if (!status) { + status = true; + psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find %s value for %s",label,colname); + } + if ((value < max_value)&&(value > min_value)) { + cdf = ((value - min_value) * max_cdf + (max_value - value) * min_cdf) / (max_value - min_value); + // fprintf(stderr,"value: %f %f %f %f %f %f %d %d %f\n",value,quality,min_cdf,min_value,max_cdf,max_value, value < max_value, value > min_value,cdf); + if (quality > cdf) { + quality = cdf; + done = 1; + i = 25; + } + } + min_value = max_value; + min_cdf = max_cdf; + } + if (done) { + continue; + } + // Reached end of array without finding the value; + if (value > min_value) { + if (quality > min_cdf) { + quality = 1.0; + } + } + + + } // End CDF rule parsing } bool success; @@ -695,4 +772,17 @@ return(false); } + success = psMetadataAddF32(tableRow,PS_LIST_TAIL,"QUALITY",0,"",quality); + if (!success) { + psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); + psFree(colNames); + psFree(colTypes); + psFree(contents); + psFree(query); + psFree(outTable); + psFree(statsoutput); + psFree(statIter); + psFree(tableRow); + return(false); + } // Add the row to the table array. Index: branches/czw_branch/20101203/ippTools/src/magicdstool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30586) @@ -1115,5 +1115,5 @@ psString queryFile = NULL; if (state) { - if (! strcmp(state, "new") ) { + if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) { queryFile = "magicdstool_revertdestreakedfile.sql"; } else if (!strcmp(state, "update")) { @@ -1724,4 +1724,5 @@ PXOPT_LOOKUP_STR(component, config->args, "-component", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-set_recoveryroot", false, false); psString query = pxDataGet("magicdstool_setfiletoupdate.sql"); @@ -1734,4 +1735,7 @@ if (label) { psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label); + } + if (recoveryroot) { + psStringAppend(&setHook, "\n , magicDSRun.recoveryroot = '%s'", recoveryroot); } Index: branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c (revision 30586) @@ -230,4 +230,5 @@ psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL); + psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_recoveryroot", 0, "define new recovery directory", NULL); // -destreakedfile Index: branches/czw_branch/20101203/ippTools/src/magictool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magictool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/magictool.c (revision 30586) @@ -49,4 +49,7 @@ static bool censorrunMode(pxConfig *config); static bool exposureMode(pxConfig *config); +static bool setgotocleanedMode(pxConfig *config); +static bool tocleanupMode(pxConfig *config); +static bool setworkdirstateMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString); @@ -89,4 +92,7 @@ MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); MODECASE(MAGICTOOL_MODE_EXPOSURE, exposureMode); + MODECASE(MAGICTOOL_MODE_SETGOTOCLEANED, setgotocleanedMode); + MODECASE(MAGICTOOL_MODE_TOCLEANUP, tocleanupMode); + MODECASE(MAGICTOOL_MODE_SETWORKDIRSTATE, setworkdirstateMode); default: psAbort("invalid option (this should not happen)"); @@ -621,5 +627,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0"); @@ -829,11 +835,4 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - // look for "inputs" that need to processed - psString query = pxDataGet("magictool_toprocess_inputs.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - psString whereString = NULL; if (psListLength(where->list)) { @@ -844,59 +843,11 @@ psFree(where); - psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); - - // treat limit == 0 as "no limit" - if (limit) { - // cut limit in half - // hack to prevent pending leaf nodes from blocking branch nodes - psString limitString = psDBGenerateLimitSQL((limit + 1) / 2); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereString); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - } - - if (limit) { - if (psArrayLength(output) >= limit) { - // we've found enough (note that limit was applied to the query so '> limit' won't happen) - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - return true; - } - } - - // look for tree nodes that need to be processed + // First look for branch nodes that need to be processed. + // These get priority over skycells because they are from runs + // that are already in progress and there are fewer of them. + // When we looked for skycells first we got starved. // first find incomplete magicRuns - query = pxDataGet("magictool_toprocess_runs.sql"); + psString query = pxDataGet("magictool_toprocess_runs.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -904,11 +855,10 @@ } - // we limit the query even though it is cheap (only magic_id is selected) - - // XXX: if the first 1000 unfinished magicRuns have no ready nodes - // that haven't faulted, later runs won't get returned even though - // they have work to do. When we used a limit of 100 we actually ran - // into this problem. Since we're using labels a limit of 1000 will - // probably be ok. + // Find outstanding magicRuns in new state. + // We limit the query, but this is problematic. In practice do + // we need to? + // XXX: If the first 1000 magicRuns have no branch nodes ready + // but higher runs do we they won't be noticed. + // Perhaps have this limit be an argument. { psString limitString = psDBGenerateLimitSQL( 1000 ); @@ -940,11 +890,116 @@ return false; } + if (!psArrayLength(magicRuns)) { + // no pending magicRuns nothing to do psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(magicRuns); return true; } - - query = pxDataGet("magictool_toprocess_tree.sql"); + psArray *output = psArrayAllocEmpty(100); + + { + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + for (psS64 index = 0; index < psArrayLength(magicRuns); index++) { + if (limit && (psArrayLength(output) >= limit)) { + break; + } + bool status; + psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id"); + if (!status) { + psAbort("failed to lookup value for magic_id column"); + } + + whereString = NULL; + psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); + if (!p_psDBRunQueryF(config->dbh, query, whereString )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + psS64 length = psArrayLength(magicTree); + if (!length) { + psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id); + psFree(magicTree); + continue; + } + + psHash *forest = psHashAlloc(length); + + // convert the array of metadata into a pxTree structure + for (long i = 0; i < length; i++) { + bool status; + psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); + if (!status) { + psAbort("failed to lookup value for node column"); + } + + psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); + if (!status) { + psAbort("failed to lookup value for dep column"); + } + + pxTreeBuilder(forest, node, dep, magicTree->data[i]); + + } + + // find the root of the tree + pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root")); + psFree(forest); + // pxTreePrint(stdout, root); + + // crawl through the tree and looking for nodes with children that are all + // "done" + pxTreeCrawl(root, findReadyNodes, output); + psFree(root); + psFree(magicTree); + } + + int len = psArrayLength(output); + if (len) { + if (limit) { + if (limit < psArrayLength(output)) { + // truncate the array + long arrayLength = psArrayLength(output); + for (long i = arrayLength - 1; i >= limit; i--) { + psArrayRemoveIndex(output, i); + } + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } else { + limit -= len; + } + } + } + } + + // look for "inputs" (skycells) that need to processed + query = pxDataGet("magictool_toprocess_inputs.sql"); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -952,84 +1007,44 @@ } - for (psS64 index = 0; index < psArrayLength(magicRuns); index++) { - if (limit && (psArrayLength(output) >= limit)) { - break; - } - bool status; - psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id"); - if (!status) { - psAbort("failed to lookup value for magic_id column"); - } - - whereString = NULL; - psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); - if (!p_psDBRunQueryF(config->dbh, query, whereString )) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereString); - psFree(query); - return false; - } + psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(whereString); - psArray *magicTree = p_psDBFetchResult(config->dbh); - if (!magicTree) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - psS64 length = psArrayLength(magicTree); - if (!length) { - psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id); - psFree(magicTree); - continue; - } - - psHash *forest = psHashAlloc(length); - - // convert the array of metadata into a pxTree structure - for (long i = 0; i < length; i++) { - bool status; - psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); - if (!status) { - psAbort("failed to lookup value for node column"); - } - - psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); - if (!status) { - psAbort("failed to lookup value for dep column"); - } - - pxTreeBuilder(forest, node, dep, magicTree->data[i]); - - } - - // find the root of the tree - pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root")); - psFree(forest); - // pxTreePrint(stdout, root); - - // crawl through the tree and looking for nodes with children that are all - // "done" - pxTreeCrawl(root, findReadyNodes, output); - psFree(root); - psFree(magicTree); - } - + psFree(query); + return false; + } + psFree(query); + + psArray *skycellOutput = p_psDBFetchResult(config->dbh); + if (!skycellOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + // merge the arrays so that we can print them all at once + if (psArrayLength(skycellOutput)) { + int len = psArrayLength(skycellOutput); + for (int i=0; i < len; i++) { + psArrayAdd(output, 0, skycellOutput->data[i]); + } + } + psFree(skycellOutput); if (psArrayLength(output)) { - if (limit && (limit < psArrayLength(output))) { - // truncate the array - long arrayLength = psArrayLength(output); - for (long i = arrayLength - 1; i >= limit; i--) { - psArrayRemoveIndex(output, i); - } - } - // negative simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -1037,8 +1052,8 @@ return false; } - } - + } else { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + } psFree(output); - return true; } @@ -1622,2 +1637,148 @@ return true; } +static bool setgotocleanedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "magicRun.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "magicRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-data_group", "magicRun.data_group", "LIKE"); + + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + + psString query = psStringCopy("UPDATE magicRun SET workdir_state = 'goto_cleaned'\n"); + if (set_label) { + psStringAppend(&query, ", label = '%s'", set_label); + } + // This mode doubles as a revert function for cleanup errors + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + if (!clearfault) { + psStringAppend(&query, "WHERE workdir_state = 'dirty'"); + } else { + psStringAppend(&query, "WHERE workdir_state = 'error_cleaned'"); + } + psStringAppend(&query, "\nAND (magicRun.state = 'full' OR magicRun.state = 'drop')"); + + // Require search parameters unless we're just clearing faults + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", clause); + psFree(clause); + psFree(where); + } else if ( !clearfault) { + psError(PS_ERR_UNKNOWN, false, "search parameters are required"); + psFree(where); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool setworkdirstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(workdir_state, config->args, "-set_workdir_state", true, false); + + if (strcmp(workdir_state, "cleaned") && strcmp(workdir_state, "error_cleaned")) { + psError(PS_ERR_UNKNOWN, true, "%s is not a valid value for workdir_state", workdir_state); + return false; + } + + psString query = NULL; + psStringAppend(&query, "UPDATE magicRun SET workdir_state = '%s' WHERE magic_id = %" PRId64, workdir_state, magic_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group", "magicRun.data_group", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/czw_branch/20101203/ippTools/src/magictool.h =================================================================== --- branches/czw_branch/20101203/ippTools/src/magictool.h (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/magictool.h (revision 30586) @@ -43,4 +43,7 @@ MAGICTOOL_MODE_CENSORRUN, MAGICTOOL_MODE_EXPOSURE, + MAGICTOOL_MODE_SETGOTOCLEANED, + MAGICTOOL_MODE_TOCLEANUP, + MAGICTOOL_MODE_SETWORKDIRSTATE, } MAGICtoolMode; Index: branches/czw_branch/20101203/ippTools/src/magictoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/magictoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/magictoolConfig.c (revision 30586) @@ -114,5 +114,5 @@ psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -inputs @@ -178,4 +178,26 @@ psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false); psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setgotocleaned + psMetadata *setgotocleanedArgs = psMetadataAlloc(); + psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID", 0); + psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID)", 0); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-data_group", 0, "select by data_group", NULL); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL); + psMetadataAddBool(setgotocleanedArgs, PS_LIST_TAIL, "-clearfault", 0, "clear cleanup errors", NULL); + + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setworkdirstate + psMetadata *setworkdirstateArgs = psMetadataAlloc(); + psMetadataAddS64(setworkdirstateArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID (required)", 0); + psMetadataAddStr(setworkdirstateArgs, PS_LIST_TAIL, "-set_workdir_state", 0, "new workdir_state (required)", 0); psFree(now); @@ -202,4 +224,7 @@ PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); PXOPT_ADD_MODE("-exposure", "", MAGICTOOL_MODE_EXPOSURE, exposureArgs); + PXOPT_ADD_MODE("-setgotocleaned", "", MAGICTOOL_MODE_SETGOTOCLEANED, setgotocleanedArgs); + PXOPT_ADD_MODE("-tocleanup", "", MAGICTOOL_MODE_TOCLEANUP, tocleanupArgs); + PXOPT_ADD_MODE("-setworkdirstate", "", MAGICTOOL_MODE_SETWORKDIRSTATE, setworkdirstateArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/czw_branch/20101203/ippTools/src/pstamptool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30586) @@ -134,4 +134,7 @@ PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-set_no_magic", false); + + bool need_magic = ! no_magic; if (!pstampDataStoreInsert(config->dbh, @@ -143,5 +146,6 @@ outProduct, uri, - pollInterval + pollInterval, + need_magic )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30586) @@ -51,4 +51,5 @@ psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label", 0, "define datastore label", NULL); psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60); + psMetadataAddBool(adddatastoreArgs,PS_LIST_TAIL, "-set_no_magic", 0, "set magic not required", true); // -datastore Index: branches/czw_branch/20101203/ippTools/src/pubtool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pubtool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pubtool.c (revision 30586) @@ -167,4 +167,5 @@ PXOPT_COPY_STR(config->args, diffWhere, "-filter", "rawExp.filter", "LIKE"); PXOPT_COPY_STR(config->args, diffWhere, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_STR(config->args, diffWhere, "-comment", "rawExp.comment", "LIKE"); PXOPT_COPY_S64(config->args, camWhere, "-client_id", "client_id", "=="); Index: branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c (revision 30586) @@ -66,4 +66,5 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "set and search by label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); Index: branches/czw_branch/20101203/ippTools/src/pztool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pztool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pztool.c (revision 30586) @@ -47,4 +47,7 @@ static bool advanceMode(pxConfig *config); +static bool updatepzexpMode(pxConfig *config); +static bool updatenewexpMode(pxConfig *config); + // static bool copydoneCompleteExp(pxConfig *config); static psArray *pzGetPendingCameras(pxConfig *config); @@ -83,4 +86,6 @@ MODECASE(PZTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(PZTOOL_MODE_ADVANCE, advanceMode); + MODECASE(PZTOOL_MODE_UPDATEPZEXP, updatepzexpMode); + MODECASE(PZTOOL_MODE_UPDATENEWEXP, updatenewexpMode); default: psAbort("invalid option (this should not happen)"); @@ -294,8 +299,10 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false); @@ -325,5 +332,5 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -889,4 +896,5 @@ || (strncmp(state, "stop", 5) == 0) || (strncmp(state, "reg", 4) == 0) + || (strncmp(state, "drop", 5) == 0) ) ) { @@ -905,4 +913,63 @@ } +static bool updatepzexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(camera, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope,config->args, "-telescope",true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state",true, false); + + if (!pzDownloadExpSetState(config,exp_name,camera,telescope,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope); + return false; + } + return true; +} + + +static bool updatenewexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "tmp_exp_name", "=="); + + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); + + if (strcmp(new_state, "drop") && strcmp(new_state, "run") && strcmp(new_state, "wait")) { + psError(PXTOOLS_ERR_ARGUMENTS, true, "%s is not a valid value for -set_state", new_state); + psFree(where); + return false; + } + + if (psListLength(where->list) < 1) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "-exp_name or -exp_id is required"); + return false; + } + + psString query = NULL; + psStringAppend(&query, "UPDATE newExp SET state = '%s'", new_state); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + + + #if 0 Index: branches/czw_branch/20101203/ippTools/src/pztool.h =================================================================== --- branches/czw_branch/20101203/ippTools/src/pztool.h (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pztool.h (revision 30586) @@ -36,5 +36,7 @@ PZTOOL_MODE_CLEARCOMMONFAULTS, PZTOOL_MODE_TOADVANCE, - PZTOOL_MODE_ADVANCE + PZTOOL_MODE_ADVANCE, + PZTOOL_MODE_UPDATEPZEXP, + PZTOOL_MODE_UPDATENEWEXP, } pztoolMode; Index: branches/czw_branch/20101203/ippTools/src/pztoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/pztoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/pztoolConfig.c (revision 30586) @@ -76,4 +76,6 @@ psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddTime(pendingimfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(pendingimfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0, "define exposure type", NULL); psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-desc", 0, "sort ouput in descending format", false); @@ -154,4 +156,18 @@ psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + // -updatepzexp + psMetadata *updatepzexpArgs = psMetadataAlloc(); + psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name (required)", NULL); + psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-inst", 0, "search by camera (required)", NULL); + psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope (required)", NULL); + psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state (required)", NULL); + + // -updatenewexp + psMetadata *updatenewexpArgs = psMetadataAlloc(); + psMetadataAddS64(updatenewexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + + psMetadataAddStr(updatenewexpArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(updatenewexpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state (required)", NULL); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -169,4 +185,6 @@ PXOPT_ADD_MODE("-toadvance", "", PZTOOL_MODE_TOADVANCE, toadvanceArgs); PXOPT_ADD_MODE("-advance", "", PZTOOL_MODE_ADVANCE, advanceArgs); + PXOPT_ADD_MODE("-updatepzexp", "", PZTOOL_MODE_UPDATEPZEXP, updatepzexpArgs); + PXOPT_ADD_MODE("-updatenewexp", "", PZTOOL_MODE_UPDATENEWEXP,updatenewexpArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30586) @@ -51,4 +51,6 @@ static bool updatebyqueryMode(pxConfig *config); +static bool checkstatusMode(pxConfig *config); + static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -74,6 +76,6 @@ // imfile MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); - MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); - MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); + MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); + MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); @@ -90,4 +92,5 @@ MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpMode); MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); + MODECASE(REGTOOL_MODE_CHECKSTATUS, checkstatusMode); MODECASE(REGTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(REGTOOL_MODE_IMPORTRUN, importrunMode); @@ -95,5 +98,5 @@ psAbort("invalid option (this should not happen)"); } - + psTrace("regtool",9,"Attempting to free config\n"); psFree(config); pmConfigDone(); @@ -180,4 +183,7 @@ } + + + static bool checkburntoolimfileMode(pxConfig *config) { @@ -187,5 +193,6 @@ PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); - PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", true, false); + PXOPT_LOOKUP_STR(dateobs_end, config->args, "-dateobs_end", true, false); PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); // optional @@ -201,10 +208,18 @@ psFree(query); query = rep; + + // convert regular class_id format to summitImfile.class_id format + rep = psStringCopy(class_id); + psFree(class_id); + class_id = rep; + + psStringSubstitute(&class_id,"ota","XY"); psStringSubstitute(&query,exp_name,"@EXP_NAME@"); psStringSubstitute(&query,class_id,"@CLASS_ID@"); - psStringSubstitute(&query,date,"@DATE@"); - - fprintf(stderr,"%s",query); + psStringSubstitute(&query,dateobs_begin,"@DATEOBS_BEGIN@"); + psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@"); + + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -215,5 +230,5 @@ } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -227,10 +242,9 @@ psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - + return false; } if (!psArrayLength(output)) { psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); return true; } @@ -242,6 +256,10 @@ psMetadata *row = output->data[i]; + if (strcasecmp(psMetadataLookupStr(NULL,row,"download_state"),"drop") == 0) { + continue; + } + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| - (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { ok_to_burn = false; } @@ -255,5 +273,5 @@ already_burned = false; } - + if (previous_uri) { psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); @@ -264,5 +282,5 @@ previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); } - + // negate simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { @@ -271,10 +289,10 @@ return false; } - + psFree(output); - + return(true); - + } @@ -284,5 +302,6 @@ // required - PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", true, false); + PXOPT_LOOKUP_STR(dateobs_end, config->args, "-dateobs_end", true, false); PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); // optional @@ -297,8 +316,9 @@ psFree(query); query = rep; - - psStringSubstitute(&query,date,"@DATE@"); - - // fprintf(stderr,"%s",query); + + psStringSubstitute(&query,dateobs_begin,"@DATEOBS_BEGIN@"); + psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@"); + + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -309,5 +329,5 @@ } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -321,7 +341,9 @@ psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - + return false; } + + // fprintf (stderr, "found %ld rows\n", output->n); if (!psArrayLength(output)) { psTrace("regtool", PS_LOG_INFO, "no rows found"); @@ -337,13 +359,26 @@ psString this_class_id = NULL; psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. - + for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; + if (strcasecmp(psMetadataLookupStr(NULL,row,"download_state"),"drop") == 0) { + continue; + } + + bool status = false; + char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id"); + if (!status) { + // fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name")); + continue; + } + // Add the information about this row and the previous, if it exists. // Write the class_id stuff for debugging. - this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&this_class_id,"XY","ota"); + // this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + this_class_id = psStringCopy(tmp_id); + psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id); + if (previous_class_id) { psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); @@ -353,10 +388,21 @@ continue; } + + if (0 && !strcmp(this_class_id, "ota44")) { + printf("STAT 1: %s (%d %d) %d %d %d\n", + this_class_id, + ok_to_burn, already_burned, + psMetadataLookupS32(NULL,row,"burntool_state"), + psMetadataLookupS32(NULL,row,"is_registered"), + psMetadataLookupS32(NULL,row,"is_downloaded")); + } + // Determine if we've crossed a class_id boundary, as this resets the bits. if (previous_class_id) { if (strcmp(this_class_id,previous_class_id) != 0) { - ok_to_burn = true; - already_burned = true; - previous_uri = NULL; + ok_to_burn = true; + already_burned = true; + previous_uri = NULL; + // fprintf (stderr, "crossed boundary: %s : %s to %s\n", psMetadataLookupStr(NULL,row,"exp_name"), previous_class_id, this_class_id); } } @@ -364,5 +410,5 @@ // Write the URIs as well. this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging. - psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); + psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); if (previous_uri) { psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); @@ -372,11 +418,11 @@ // Convert bits of the SQL query into booleans describing the data state if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| - (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { - // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); - ok_to_burn = false; + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id); + ok_to_burn = false; } if (already_burned == false) { - // printf("already_burned looks false %s %s\n",this_uri,this_class_id); - ok_to_burn = false; + // printf("already_burned looks false %s %s\n",this_uri,this_class_id); + ok_to_burn = false; } if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { @@ -391,44 +437,54 @@ // Check the uri for this exposure if (!this_uri) { - ok_to_burn = false; - already_burned = false; - - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; - } - - // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); - - // If the state of this imfile is not "pending_burntool" then we can't burn it. + ok_to_burn = false; + already_burned = false; + + // fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id); + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id")); + psStringSubstitute(&previous_class_id,"ota","XY"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; + } + + if (0 && !strcmp(this_class_id, "ota44")) { + printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded")); + } + // If the state of this imfile is not "pending_burntool" then we can't burn it. if (!ignore_state) { psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); if (!imfile_state) { // imfile state is NULL, so we probably aren't registered. - ok_to_burn = false; - - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; + ok_to_burn = false; + + // fprintf (stderr, "missing imfile_state: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id); + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id")); + psStringSubstitute(&previous_class_id,"ota","XY"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; } - if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states - // Save this round for next round. - psFree(previous_class_id); - psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - continue; + if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states + + if (0 && strcmp(imfile_state, "full")) { + fprintf (stderr, "not pending nor full: %s %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id, imfile_state); + } + + // Save this round for next round. + psFree(previous_class_id); + psFree(previous_uri); + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id")); + psStringSubstitute(&previous_class_id,"ota","XY"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + continue; } } - + // Determine if we've already suggested an entry for this ota, and if not, copy this // suggestion into our output result list. @@ -437,7 +493,7 @@ psFree(previous_class_id); psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id")); + psStringSubstitute(&previous_class_id,"ota","XY"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. continue; } @@ -448,9 +504,9 @@ psFree(previous_class_id); psFree(previous_uri); - previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); - psStringSubstitute(&previous_class_id,"XY","ota"); - previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. - } - + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id")); + psStringSubstitute(&previous_class_id,"ota","XY"); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round. + } + // negate simple so the default is true if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { @@ -460,14 +516,14 @@ return false; } - + psFree(output); psFree(results); - + return(true); - -} - - + +} + + static bool addprocessedimfileMode(pxConfig *config) @@ -545,5 +601,5 @@ PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false); - + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); @@ -618,6 +674,6 @@ bytes, md5sum, - video_cells, - 0 // burntool_state + 0, // burntool_state + video_cells )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -771,8 +827,4 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); - } else { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; } psFree(where); @@ -806,9 +858,9 @@ PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); - if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) { psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); return false; } - if ((fault != INT16_MAX) && isfinite(burntool_state)) { + if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) { psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; @@ -1312,8 +1364,8 @@ if (!pxchipQueueByExpTag(config, exp_id, - chip_workdir, + chip_workdir, label, - data_group, - dist_group, + data_group, + dist_group, reduction, NULL, // expgroup @@ -1831,4 +1883,81 @@ } +static bool checkstatusMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + // Conditionally required + PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", false, false); + PXOPT_LOOKUP_STR(dateobs_end, config->args, "-dateobs_end", false, false); + PXOPT_LOOKUP_STR(date, config->args, "-date", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_checkstatus.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + psStringSubstitute(&class_id,"ota","XY"); + psStringSubstitute(&query,class_id,"@CLASS_ID@"); + + if (!date) { + if (!dateobs_begin || !dateobs_end) { + psError(PXTOOLS_ERR_CONFIG, false, "Either -date or -dateobs_begin -dateobs_end is required"); + psFree(query); + return false; + } + psStringAppend(&query," AND summitExp.dateobs >= '%s' AND summitExp.dateobs <= '%s' ", + dateobs_begin,dateobs_end); + } + else { + psStringAppend(&query," AND summitExp.dateobs >= '%sT00:00:00' AND summitExp.dateobs <= '%sT23:59:59' ", + date,date); + } + + psStringAppend(&query," ORDER BY summitExp.dateobs "); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "regCheckStatus", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return(true); +} + bool exportrunMode(pxConfig *config) { Index: branches/czw_branch/20101203/ippTools/src/regtool.h =================================================================== --- branches/czw_branch/20101203/ippTools/src/regtool.h (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/regtool.h (revision 30586) @@ -41,4 +41,5 @@ REGTOOL_MODE_PENDINGCOMPRESSIMFILE, REGTOOL_MODE_FINISHCOMPRESSEXP, + REGTOOL_MODE_CHECKSTATUS, REGTOOL_MODE_EXPORTRUN, REGTOOL_MODE_IMPORTRUN Index: branches/czw_branch/20101203/ippTools/src/regtoolConfig.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 30586) @@ -57,5 +57,6 @@ ADD_OPT(Str, checkburntoolimfileArgs, "-exp_name", "define the exp_name (required)", NULL); ADD_OPT(Str, checkburntoolimfileArgs, "-class_id", "define class ID (required)", NULL); - ADD_OPT(Str, checkburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-dateobs_begin", "set the earliest summit dateobs to consider (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-dateobs_end", "set the latest summit dateobs to consider (required)", NULL); ADD_OPT(S32, checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); ADD_OPT(Str, checkburntoolimfileArgs, "-inst", "define the camera name", NULL); @@ -65,10 +66,11 @@ // -pendingburntoolimfile psMetadata *pendingburntoolimfileArgs = psMetadataAlloc(); - ADD_OPT(Str, pendingburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(Str, pendingburntoolimfileArgs, "-dateobs_begin", "set the earliest summit dateobs to consider (required)", NULL); + ADD_OPT(Str, pendingburntoolimfileArgs, "-dateobs_end", "set the latest summit dateobs to consider (required)", NULL); ADD_OPT(S32, pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple", "use the simple output format", false); ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state", "ignore the data_state when deciding what to work on", false); ADD_OPT(U64, pendingburntoolimfileArgs, "-limit", "limit result set to N items", 0); - + // -addprocessedimfile psMetadata *addprocessedimfileArgs = psMetadataAlloc(); @@ -178,4 +180,6 @@ ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_begin", "search by exposure ID", 0); ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_end", "search by exposure ID", 0); + // This argument is not used but is needed because the task uses add.poll.args + ADD_OPT(U64, revertprocessedimfileArgs, "-limit", "for compatability not used", 0); // -updateprocessedimfile @@ -415,4 +419,12 @@ psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -checkstatus + psMetadata *checkstatusArgs = psMetadataAlloc(); + ADD_OPT(Str, checkstatusArgs, "-dateobs_begin", "set the earlist summit dateobs to consider", NULL); + ADD_OPT(Str, checkstatusArgs, "-dateobs_end", "set the latest summit dateobs to consider", NULL); + ADD_OPT(Str, checkstatusArgs, "-date", "use default observing extent and look over entire night", NULL); + ADD_OPT(Str, checkstatusArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Bool, checkstatusArgs, "-simple", "use the simple output format", false); psMetadata *argSets = psMetadataAlloc(); @@ -435,4 +447,5 @@ PXOPT_ADD_MODE("-pendingcompressimfile", "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs); PXOPT_ADD_MODE("-finishcompressexp", "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs); + PXOPT_ADD_MODE("-checkstatus", "", REGTOOL_MODE_CHECKSTATUS, checkstatusArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", REGTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/czw_branch/20101203/ippTools/src/stacktool.c =================================================================== --- branches/czw_branch/20101203/ippTools/src/stacktool.c (revision 30118) +++ branches/czw_branch/20101203/ippTools/src/stacktool.c (revision 30586) @@ -497,5 +497,4 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(output); - psFree(run); psFree(insert); psFree(list); @@ -517,5 +516,4 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(output); - psFree(run); psFree(insert); psFree(list);