Index: /tags/ipp-20101215/ippTools/share/Makefile.am =================================================================== --- /tags/ipp-20101215/ippTools/share/Makefile.am (revision 30416) +++ /tags/ipp-20101215/ippTools/share/Makefile.am (revision 30417) @@ -246,4 +246,6 @@ magictool_exposure.sql \ magictool_setfull.sql \ + magictool_setgotocleaned.sql \ + magictool_tocleanup.sql \ magicdstool_clearstatefaults.sql \ magicdstool_change_file_data_state.sql \ @@ -333,4 +335,5 @@ regtool_updatebyqueryimfile.sql \ regtool_pendingburntoolimfile.sql \ + regtool_checkstatus.sql \ stacktool_associationdefine_select.sql \ stacktool_definebyquery_insert.sql \ Index: /tags/ipp-20101215/ippTools/share/magictool_setgotocleaned.sql =================================================================== --- /tags/ipp-20101215/ippTools/share/magictool_setgotocleaned.sql (revision 30417) +++ /tags/ipp-20101215/ippTools/share/magictool_setgotocleaned.sql (revision 30417) @@ -0,0 +1,4 @@ +UPDATE magicRun + SET workdir_state = 'goto_cleaned' +WHERE workdir_state = 'dirty' + Index: /tags/ipp-20101215/ippTools/share/magictool_tocleanup.sql =================================================================== --- /tags/ipp-20101215/ippTools/share/magictool_tocleanup.sql (revision 30417) +++ /tags/ipp-20101215/ippTools/share/magictool_tocleanup.sql (revision 30417) @@ -0,0 +1,10 @@ +SELECT magic_id, + magicRun.exp_id, + rawExp.camera, + magicRun.workdir, + IFNULL(priority, 10000) as priority +FROM magicRun + JOIN rawExp USING(exp_id) + LEFT JOIN Label ON magicRun.label = Label.label +WHERE magicRun.workdir_state = 'goto_cleaned' + AND (Label.active OR Label.active IS NULL) Index: /tags/ipp-20101215/ippTools/share/regtool_checkstatus.sql =================================================================== --- /tags/ipp-20101215/ippTools/share/regtool_checkstatus.sql (revision 30417) +++ /tags/ipp-20101215/ippTools/share/regtool_checkstatus.sql (revision 30417) @@ -0,0 +1,29 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + summitImfile.class_id AS summit_class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM summitExp + JOIN summitImfile USING(exp_name) + LEFT JOIN pzDownloadExp USING(exp_name) + LEFT JOIN newExp ON summitExp.exp_name = tmp_exp_name + LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id) + LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id + LEFT JOIN rawImfile ON (rawImfile.exp_id = rawExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitImfile.class_id = '@CLASS_ID@' Index: /tags/ipp-20101215/ippTools/src/magictool.c =================================================================== --- /tags/ipp-20101215/ippTools/src/magictool.c (revision 30416) +++ /tags/ipp-20101215/ippTools/src/magictool.c (revision 30417) @@ -49,4 +49,7 @@ static bool censorrunMode(pxConfig *config); static bool exposureMode(pxConfig *config); +static bool setgotocleanedMode(pxConfig *config); +static bool tocleanupMode(pxConfig *config); +static bool setworkdirstateMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString); @@ -89,4 +92,7 @@ MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); MODECASE(MAGICTOOL_MODE_EXPOSURE, exposureMode); + MODECASE(MAGICTOOL_MODE_SETGOTOCLEANED, setgotocleanedMode); + MODECASE(MAGICTOOL_MODE_TOCLEANUP, tocleanupMode); + MODECASE(MAGICTOOL_MODE_SETWORKDIRSTATE, setworkdirstateMode); default: psAbort("invalid option (this should not happen)"); @@ -837,8 +843,8 @@ psFree(where); - - // First look for tree nodes that need to be processed + // First look for branch nodes that need to be processed. // These get priority over skycells because they are from runs - // that are already in progress + // that are already in progress and there are fewer of them. + // When we looked for skycells first we got starved. // first find incomplete magicRuns @@ -849,12 +855,10 @@ } - // we limit the query even though it is cheap (only magic_id is selected) - - // XXX: The following comment no longer applies - // XXX: if the first 1000 unfinished magicRuns have no ready nodes - // that haven't faulted, later runs won't get returned even though - // they have work to do. When we used a limit of 100 we actually ran - // into this problem. Since we're using labels a limit of 1000 will - // probably be ok. + // Find outstanding magicRuns in new state. + // We limit the query, but this is problematic. In practice do + // we need to? + // XXX: If the first 1000 magicRuns have no branch nodes ready + // but higher runs do we they won't be noticed. + // Perhaps have this limit be an argument. { psString limitString = psDBGenerateLimitSQL( 1000 ); @@ -886,13 +890,13 @@ return false; } + if (!psArrayLength(magicRuns)) { - // no magicRuns pending. Nothing to do + // no pending magicRuns nothing to do psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(magicRuns); return true; } - psArray *output = psArrayAllocEmpty(100); - // the following is indented because I removed a conditional and I decided not to shift over + + { query = pxDataGet("magictool_toprocess_tree.sql"); if (!query) { @@ -994,7 +998,7 @@ } } - // the above code is indented because I removed a conditional block and I decided not to shift it over - - // look for "inputs" that need to processed + } + + // look for "inputs" (skycells) that need to processed query = pxDataGet("magictool_toprocess_inputs.sql"); if (!query) { @@ -1633,2 +1637,148 @@ return true; } +static bool setgotocleanedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "magicRun.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "magicRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-data_group", "magicRun.data_group", "LIKE"); + + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + + psString query = psStringCopy("UPDATE magicRun SET workdir_state = 'goto_cleaned'\n"); + if (set_label) { + psStringAppend(&query, ", label = '%s'", set_label); + } + // This mode doubles as a revert function for cleanup errors + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + if (!clearfault) { + psStringAppend(&query, "WHERE workdir_state = 'dirty'"); + } else { + psStringAppend(&query, "WHERE workdir_state = 'error_cleaned'"); + } + psStringAppend(&query, "\nAND (magicRun.state = 'full' OR magicRun.state = 'drop')"); + + // Require search parameters unless we're just clearing faults + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", clause); + psFree(clause); + psFree(where); + } else if ( !clearfault) { + psError(PS_ERR_UNKNOWN, false, "search parameters are required"); + psFree(where); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool setworkdirstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); + PXOPT_LOOKUP_STR(workdir_state, config->args, "-set_workdir_state", true, false); + + if (strcmp(workdir_state, "cleaned") && strcmp(workdir_state, "error_cleaned")) { + psError(PS_ERR_UNKNOWN, true, "%s is not a valid value for workdir_state", workdir_state); + return false; + } + + psString query = NULL; + psStringAppend(&query, "UPDATE magicRun SET workdir_state = '%s' WHERE magic_id = %" PRId64, workdir_state, magic_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group", "magicRun.data_group", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, magic_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /tags/ipp-20101215/ippTools/src/magictool.h =================================================================== --- /tags/ipp-20101215/ippTools/src/magictool.h (revision 30416) +++ /tags/ipp-20101215/ippTools/src/magictool.h (revision 30417) @@ -43,4 +43,7 @@ MAGICTOOL_MODE_CENSORRUN, MAGICTOOL_MODE_EXPOSURE, + MAGICTOOL_MODE_SETGOTOCLEANED, + MAGICTOOL_MODE_TOCLEANUP, + MAGICTOOL_MODE_SETWORKDIRSTATE, } MAGICtoolMode; Index: /tags/ipp-20101215/ippTools/src/magictoolConfig.c =================================================================== --- /tags/ipp-20101215/ippTools/src/magictoolConfig.c (revision 30416) +++ /tags/ipp-20101215/ippTools/src/magictoolConfig.c (revision 30417) @@ -178,4 +178,26 @@ psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false); psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setgotocleaned + psMetadata *setgotocleanedArgs = psMetadataAlloc(); + psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID", 0); + psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID)", 0); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-data_group", 0, "select by data_group", NULL); + psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL); + psMetadataAddBool(setgotocleanedArgs, PS_LIST_TAIL, "-clearfault", 0, "clear cleanup errors", NULL); + + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -setworkdirstate + psMetadata *setworkdirstateArgs = psMetadataAlloc(); + psMetadataAddS64(setworkdirstateArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID (required)", 0); + psMetadataAddStr(setworkdirstateArgs, PS_LIST_TAIL, "-set_workdir_state", 0, "new workdir_state (required)", 0); psFree(now); @@ -202,4 +224,7 @@ PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); PXOPT_ADD_MODE("-exposure", "", MAGICTOOL_MODE_EXPOSURE, exposureArgs); + PXOPT_ADD_MODE("-setgotocleaned", "", MAGICTOOL_MODE_SETGOTOCLEANED, setgotocleanedArgs); + PXOPT_ADD_MODE("-tocleanup", "", MAGICTOOL_MODE_TOCLEANUP, tocleanupArgs); + PXOPT_ADD_MODE("-setworkdirstate", "", MAGICTOOL_MODE_SETWORKDIRSTATE, setworkdirstateArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {