Index: /tags/ipp-20101029/ippTools/share/Makefile.am =================================================================== --- /tags/ipp-20101029/ippTools/share/Makefile.am (revision 29717) +++ /tags/ipp-20101029/ippTools/share/Makefile.am (revision 29718) @@ -21,4 +21,8 @@ bgtool_advancechip.sql \ bgtool_advancewarp.sql \ + bgtool_change_chiprun_state.sql \ + bgtool_change_imfile_data_state.sql \ + bgtool_change_warprun_state.sql \ + bgtool_change_skyfile_data_state.sql \ bgtool_chipinputs.sql \ bgtool_chip.sql \ @@ -29,4 +33,8 @@ bgtool_listchip.sql \ bgtool_listwarp.sql \ + bgtool_pendingcleanupchiprun.sql \ + bgtool_pendingcleanupchipimfile.sql \ + bgtool_pendingcleanupwarprun.sql \ + bgtool_pendingcleanupwarpskyfile.sql \ bgtool_revertchip.sql \ bgtool_revertwarp.sql \ @@ -377,4 +385,5 @@ warptool_tooverlap.sql \ warptool_towarped.sql \ + warptool_towarped_labels.sql \ warptool_updateskyfile.sql \ warptool_warped.sql \ Index: /tags/ipp-20101029/ippTools/share/bgtool_change_chiprun_state.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_change_chiprun_state.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_change_chiprun_state.sql (revision 29718) @@ -0,0 +1,10 @@ +UPDATE chipBackgroundRun +SET chipBackgroundRun.state = '%s', + chipBackgroundRun.magicked = IF(chipBackgroundRun.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld +AND (SELECT COUNT(class_id) + FROM chipBackgroundImfile + WHERE + chipBackgroundImfile.chip_bg_id = chipBackgroundRun.chip_bg_id + AND data_state != '%s' + ) = 0 Index: /tags/ipp-20101029/ippTools/share/bgtool_change_imfile_data_state.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_change_imfile_data_state.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_change_imfile_data_state.sql (revision 29718) @@ -0,0 +1,5 @@ +UPDATE chipBackgroundImfile + JOIN chipBackgroundRun USING(chip_bg_id) +SET data_state = '%s', + chipBackgroundImfile.magicked = IF(chipBackgroundImfile.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld AND class_id = '%s' Index: /tags/ipp-20101029/ippTools/share/bgtool_change_skyfile_data_state.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_change_skyfile_data_state.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_change_skyfile_data_state.sql (revision 29718) @@ -0,0 +1,5 @@ +UPDATE warpBackgroundSkyfile + JOIN warpBackgroundRun USING(warp_bg_id) +SET data_state = '%s', + warpBackgroundSkyfile.magicked = IF(warpBackgroundSkyfile.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld AND skycell_id = '%s' Index: /tags/ipp-20101029/ippTools/share/bgtool_change_warprun_state.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_change_warprun_state.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_change_warprun_state.sql (revision 29718) @@ -0,0 +1,10 @@ +UPDATE warpBackgroundRun +SET state = '%s', + warpBackgroundRun.magicked = IF(warpBackgroundRun.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld +AND (SELECT COUNT(skycell_id) + FROM warpBackgroundSkyfile + WHERE + warpBackgroundSkyfile.warp_bg_id = warpBackgroundRun.warp_bg_id + AND data_state != '%s' + ) = 0 Index: /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchipimfile.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchipimfile.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchipimfile.sql (revision 29718) @@ -0,0 +1,15 @@ +SELECT + chipBackgroundImfile.*, + chipBackgroundRun.state, + chipBackgroundRun.workdir, + chipBackgroundRun.label +FROM chipBackgroundRun +JOIN chipBackgroundImfile + USING(chip_bg_id) +WHERE + ((chipBackgroundRun.state = 'goto_cleaned' AND (chipBackgroundImfile.data_state = 'full' + OR chipBackgroundImfile.data_state = 'update')) +OR + (chipBackgroundRun.state = 'goto_scrubbed' AND chipBackgroundImfile.data_state != 'scrubbed') +OR + (chipBackgroundRun.state = 'goto_purged' AND chipBackgroundImfile.data_state != 'purged')) Index: /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchiprun.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchiprun.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupchiprun.sql (revision 29718) @@ -0,0 +1,12 @@ +SELECT + chipBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM chipBackgroundRun + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (chipBackgroundRun.state = 'goto_cleaned' + OR chipBackgroundRun.state = 'goto_purged' + OR chipBackgroundRun.state = 'goto_scrubbed' + ) + Index: /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarprun.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarprun.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarprun.sql (revision 29718) @@ -0,0 +1,13 @@ +SELECT warpBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM warpBackgroundRun + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (warpBackgroundRun.state = 'goto_cleaned' + OR warpBackgroundRun.state = 'goto_purged' + OR warpBackgroundRun.state = 'goto_scrubbed') + Index: /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql (revision 29718) @@ -0,0 +1,15 @@ +SELECT + warpBackgroundSkyfile.*, + warpBackgroundRun.state, + warpBackgroundRun.workdir, + warpBackgroundRun.label +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile + USING(warp_bg_id) +WHERE + ((warpBackgroundRun.state = 'goto_cleaned' AND (warpBackgroundSkyfile.data_state = 'full' + OR warpBackgroundSkyfile.data_state = 'update')) +OR + (warpBackgroundRun.state = 'goto_scrubbed' AND warpBackgroundSkyfile.data_state != 'scrubbed') +OR + (warpBackgroundRun.state = 'goto_purged' AND warpBackgroundSkyfile.data_state != 'purged')) Index: /tags/ipp-20101029/ippTools/share/warptool_definebyquery.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/warptool_definebyquery.sql (revision 29717) +++ /tags/ipp-20101029/ippTools/share/warptool_definebyquery.sql (revision 29718) @@ -29,4 +29,5 @@ JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) -WHERE camRun.state = 'full' +WHERE fakeRun.state = 'full' +AND camRun.state = 'full' AND chipRun.state = 'full' Index: /tags/ipp-20101029/ippTools/share/warptool_towarped_labels.sql =================================================================== --- /tags/ipp-20101029/ippTools/share/warptool_towarped_labels.sql (revision 29718) +++ /tags/ipp-20101029/ippTools/share/warptool_towarped_labels.sql (revision 29718) @@ -0,0 +1,15 @@ +SELECT DISTINCT + warpRun.label, + Label.label AS label_label, + IFNULL(Label.priority, 10000) AS priority, + count(warpRun.warp_id) + FROM warpRun + LEFT JOIN Label ON warpRun.label = Label.label + WHERE + (warpRun.state = 'new' OR warpRun.state = 'update') + AND (Label.active OR Label.active IS NULL) + -- where hook %s + group by warpRun.label + ORDER BY priority DESC, warp_id + limit 100 + Index: /tags/ipp-20101029/ippTools/src/bgtool.c =================================================================== --- /tags/ipp-20101029/ippTools/src/bgtool.c (revision 29717) +++ /tags/ipp-20101029/ippTools/src/bgtool.c (revision 29718) @@ -49,12 +49,20 @@ static bool revertwarpMode(pxConfig *config); static bool listwarpMode(pxConfig *config); -static bool tocleanchipMode(pxConfig *config); -static bool cleanedchipMode(pxConfig *config); -static bool tocleanwarpMode(pxConfig *config); -static bool cleanedwarpMode(pxConfig *config); + +static bool pendingcleanupchiprunMode(pxConfig *config); +static bool pendingcleanupchipimfileMode(pxConfig *config); +static bool tocleanedchipimfileMode(pxConfig *config); +static bool updatechipimfileMode(pxConfig *config); + +static bool pendingcleanupwarprunMode(pxConfig *config); +static bool pendingcleanupwarpskyfileMode(pxConfig *config); +static bool tocleanedwarpskyfileMode(pxConfig *config); + static bool exportchipMode(pxConfig *config); static bool importchipMode(pxConfig *config); static bool exportwarpMode(pxConfig *config); static bool importwarpMode(pxConfig *config); + +static bool validDataState(psString data_state); // Tables to import/export @@ -112,8 +120,11 @@ MODECASE(BGTOOL_MODE_REVERTWARP, revertwarpMode); MODECASE(BGTOOL_MODE_LISTWARP, listwarpMode); - MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode); - MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode); - MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode); - MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, pendingcleanupchiprunMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, pendingcleanupchipimfileMode); + MODECASE(BGTOOL_MODE_TOCLEANEDCHIPIMFILE, tocleanedchipimfileMode); + MODECASE(BGTOOL_MODE_UPDATECHIPIMFILE, updatechipimfileMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPRUN, pendingcleanupwarprunMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, pendingcleanupwarpskyfileMode); + MODECASE(BGTOOL_MODE_TOCLEANEDWARPSKYFILE, tocleanedwarpskyfileMode); MODECASE(BGTOOL_MODE_EXPORTCHIP, exportchipMode); MODECASE(BGTOOL_MODE_IMPORTCHIP, importchipMode); @@ -914,4 +925,5 @@ return true; } + static bool listchipMode(pxConfig *config) { @@ -978,5 +990,5 @@ } -static bool tocleanchipMode(pxConfig *config) +static bool pendingcleanupchiprunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -987,6 +999,7 @@ psMetadata *where = psMetadataAlloc(); pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); - - psString query = pxDataGet("bgtool_tocleanchip.sql"); + pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "=="); + + psString query = pxDataGet("bgtool_pendingcleanupchiprun.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1035,24 +1048,17 @@ return true; } - -static bool cleanedchipMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); +static bool pendingcleanupchipimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - if (!pxIsValidCleanedState(state)) { - psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); - return false; - } - - if (!psListLength(where->list)) { - psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); - return false; - } - - psString query = pxDataGet("bgtool_cleanedchip.sql"); + psString query = pxDataGet("bgtool_pendingcleanupchipimfile.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1060,5 +1066,5 @@ } - { + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); @@ -1067,6 +1073,142 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psError(psErrorCodeLast(), false, "database error"); + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool change_imfile_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("bgtool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, class_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("bgtool_change_chiprun_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool tocleanedchipimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned"); +} + +static bool updatechipimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundImfile.chip_bg_id", "=="); + PXOPT_COPY_S64(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + + // optional + + PXOPT_LOOKUP_STR(data_state, config->args, "-set_data_state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + if (!data_state && !fault) { + psError(PS_ERR_UNKNOWN, true, "must supply either -set_fault or -set_data_state"); + return false; + } + char *sep = ""; + psString query = psStringCopy("UPDATE chipBackgroundImfile SET "); + if (data_state) { + if (!validDataState(data_state)) { + return false; + } + psStringAppend(&query, "%s data_state = '%s'", sep, data_state); + sep = ", "; + } + if (fault) { + psStringAppend(&query, "%s fault = %d", sep, fault); + sep = ", "; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); return false; } @@ -1449,5 +1591,5 @@ return true; } - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1814,5 +1956,5 @@ } -static bool tocleanwarpMode(pxConfig *config) +static bool pendingcleanupwarprunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -1824,5 +1966,5 @@ pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); - psString query = pxDataGet("bgtool_tocleanwarp.sql"); + psString query = pxDataGet("bgtool_pendingcleanupwarprun.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1871,24 +2013,17 @@ return true; } - -static bool cleanedwarpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); +static bool pendingcleanupwarpskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - if (!pxIsValidCleanedState(state)) { - psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); - return false; - } - - if (!psListLength(where->list)) { - psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); - return false; - } - - psString query = pxDataGet("bgtool_cleanedwarp.sql"); + psString query = pxDataGet("bgtool_pendingcleanupwarpskyfile.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1896,5 +2031,5 @@ } - { + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); @@ -1903,11 +2038,93 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psError(psErrorCodeLast(), false, "database error"); - return false; - } - psFree(query); - - return true; + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool change_skyfile_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id are required + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("bgtool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, skycell_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("bgtool_change_warprun_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool tocleanedwarpskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "cleaned"); } @@ -1935,2 +2152,19 @@ return importTables(config, infile, warpTables); } + +static bool validDataState(psString data_state) +{ + // NOTE: can't use pxIsValidState because + // update, scrubbed and purged are not supported for the backround stages (yet?) + if (!strcmp(data_state, "new") || + !strcmp(data_state, "full") || + !strcmp(data_state, "drop") || + !strcmp(data_state, "cleaned") || + !strcmp(data_state, "goto_cleaned") || + !strcmp(data_state, "error_cleaned")) { + return true; + } else { + psError(PS_ERR_UNKNOWN, true, "%s is not a valid value data_state", data_state); + return false; + } +} Index: /tags/ipp-20101029/ippTools/src/bgtool.h =================================================================== --- /tags/ipp-20101029/ippTools/src/bgtool.h (revision 29717) +++ /tags/ipp-20101029/ippTools/src/bgtool.h (revision 29718) @@ -35,4 +35,5 @@ BGTOOL_MODE_REVERTCHIP, BGTOOL_MODE_LISTCHIP, + BGTOOL_MODE_UPDATECHIPIMFILE, // Warp stage BGTOOL_MODE_DEFINEWARP, @@ -45,9 +46,12 @@ BGTOOL_MODE_REVERTWARP, BGTOOL_MODE_LISTWARP, + BGTOOL_MODE_UPDATEWARPSKYFILE, // Cleanups - BGTOOL_MODE_TOCLEANCHIP, - BGTOOL_MODE_CLEANEDCHIP, - BGTOOL_MODE_TOCLEANWARP, - BGTOOL_MODE_CLEANEDWARP, + BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, + BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, + BGTOOL_MODE_TOCLEANEDCHIPIMFILE, + BGTOOL_MODE_PENDINGCLEANUPWARPRUN, + BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, + BGTOOL_MODE_TOCLEANEDWARPSKYFILE, // Exporting BGTOOL_MODE_EXPORTCHIP, Index: /tags/ipp-20101029/ippTools/src/bgtoolConfig.c =================================================================== --- /tags/ipp-20101029/ippTools/src/bgtoolConfig.c (revision 29717) +++ /tags/ipp-20101029/ippTools/src/bgtoolConfig.c (revision 29718) @@ -192,14 +192,30 @@ psMetadataAddU64(listchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -tocleanchip - psMetadata *tocleanchipArgs = psMetadataAlloc(); - psMetadataAddStr(tocleanchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(tocleanchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(tocleanchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -cleanedchip - psMetadata *cleanedchipArgs = psMetadataAlloc(); - psMetadataAddS64(cleanedchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); - psMetadataAddStr(cleanedchipArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + // -pendingcleanupchiprun + psMetadata *pendingcleanupchiprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for specified label", NULL); + psMetadataAddStr(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "list runs for specified data_group", NULL); + psMetadataAddBool(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupchipimfile + psMetadata *pendingcleanupchipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id (required)", 0); + psMetadataAddBool(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedchipimfile + psMetadata *tocleanedchipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedchipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(tocleanedchipimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddBool(tocleanedchipimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanedchipimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -updatechipimfile + psMetadata *updatechipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatechipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(updatechipimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id (required)", NULL); + psMetadataAddStr(updatechipimfileArgs, PS_LIST_TAIL, "-set_data_state", 0, "new value for data_state", NULL); + psMetadataAddS16(updatechipimfileArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); // -exportchip @@ -361,14 +377,29 @@ psMetadataAddU64(listwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -tocleanwarp - psMetadata *tocleanwarpArgs = psMetadataAlloc(); - psMetadataAddStr(tocleanwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(tocleanwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(tocleanwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -cleanedwarp - psMetadata *cleanedwarpArgs = psMetadataAlloc(); - psMetadataAddS64(cleanedwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); - psMetadataAddStr(cleanedwarpArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + // -pendingcleanupwarprun + psMetadata *pendingcleanupwarprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddStr(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "list blocks for specified data_group", NULL); + psMetadataAddBool(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupwarpskyfile + psMetadata *pendingcleanupwarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0); + psMetadataAddBool(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanwarpskyfile + psMetadata *tocleanedwarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0); + psMetadataAddStr(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id (required)", 0); + psMetadataAddStr(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -updatewarpskyfile + psMetadata *updatewarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatewarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(updatewarpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id (required)", NULL); + psMetadataAddStr(updatewarpskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "new value for data_state", NULL); + psMetadataAddS16(updatewarpskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); // -exportwarp @@ -397,6 +428,8 @@ PXOPT_ADD_MODE("-revertchip", "", BGTOOL_MODE_REVERTCHIP, revertchipArgs); PXOPT_ADD_MODE("-listchip", "", BGTOOL_MODE_LISTCHIP, listchipArgs); - PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs); - PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs); + PXOPT_ADD_MODE("-pendingcleanupchiprun", "", BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, pendingcleanupchiprunArgs); + PXOPT_ADD_MODE("-pendingcleanupchipimfile", "", BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, pendingcleanupchipimfileArgs); + PXOPT_ADD_MODE("-tocleanedchipimfile", "", BGTOOL_MODE_TOCLEANEDCHIPIMFILE, tocleanedchipimfileArgs); + PXOPT_ADD_MODE("-updatechipimfile", "", BGTOOL_MODE_UPDATECHIPIMFILE, updatechipimfileArgs); PXOPT_ADD_MODE("-exportchip", "", BGTOOL_MODE_EXPORTCHIP, exportchipArgs); PXOPT_ADD_MODE("-importchip", "", BGTOOL_MODE_IMPORTCHIP, importchipArgs); @@ -411,6 +444,8 @@ PXOPT_ADD_MODE("-revertwarp", "", BGTOOL_MODE_REVERTWARP, revertwarpArgs); PXOPT_ADD_MODE("-listwarp", "", BGTOOL_MODE_LISTWARP, listwarpArgs); - PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs); - PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs); + PXOPT_ADD_MODE("-pendingcleanupwarprun", "", BGTOOL_MODE_PENDINGCLEANUPWARPRUN, pendingcleanupwarprunArgs); + PXOPT_ADD_MODE("-pendingcleanupwarpskyfile", "", BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, pendingcleanupwarpskyfileArgs); + PXOPT_ADD_MODE("-tocleanedwarpskyfile", "", BGTOOL_MODE_TOCLEANEDWARPSKYFILE, tocleanedwarpskyfileArgs); + PXOPT_ADD_MODE("-updatewarpskyfile", "", BGTOOL_MODE_UPDATEWARPSKYFILE, updatewarpskyfileArgs); PXOPT_ADD_MODE("-exportwarp", "", BGTOOL_MODE_EXPORTWARP, exportwarpArgs); PXOPT_ADD_MODE("-importwarp", "", BGTOOL_MODE_IMPORTWARP, importwarpArgs); Index: /tags/ipp-20101029/ippTools/src/warptool.c =================================================================== --- /tags/ipp-20101029/ippTools/src/warptool.c (revision 29717) +++ /tags/ipp-20101029/ippTools/src/warptool.c (revision 29718) @@ -924,18 +924,48 @@ psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_towarped.sql"); - if (!query) { + // First find Label's with warpRuns in state new in priority order + psMetadata *labelWhere = psMetadataAlloc(); + pxAddLabelSearchArgs (config, labelWhere, "-label", "warpRun.label", "=="); + + psString labelQuery = pxDataGet("warptool_towarped_labels.sql"); + if (!labelQuery) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } - - psString whereStr = psStringCopy(""); + psString labelWhereStr = psStringCopy(""); + if (psListLength(labelWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(labelWhere, NULL); + psStringAppend(&labelWhereStr, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(labelWhere); + + if (!p_psDBRunQueryF(config->dbh, labelQuery, labelWhereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(labelQuery); + return false; + } + psFree(labelQuery); + psFree(labelWhereStr); + + psArray *labelOutput = p_psDBFetchResult(config->dbh); + if (psArrayLength(labelOutput) == 0) { + psTrace("warptool", PS_LOG_INFO, "no labels with pending warps found."); + return true; + } + + psString originalQuery = pxDataGet("warptool_towarped.sql"); + if (!originalQuery) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -945,52 +975,76 @@ psFree(where); - // treat limit == 0 as "no limit" - psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id"); - if (limit) { - // We apply the limit to both sides of the UNION to avoid slow queries - // and to the query itself to satisfy the user's requested limit - psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); - psStringAppend(&query, " %s", limitString); - } - - if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + // Now loop over the labels and query each individually + for (long i = 0; i < psArrayLength(labelOutput); i++) { + psMetadata *labelRow = labelOutput->data[i]; + + bool status; + psString label = psMetadataLookupStr(&status, labelRow, "label"); + + psString query = psStringCopy(originalQuery); + + // treat limit == 0 as "no limit" + psString limitString = psStringCopy(""); + if (limit) { + // We apply the limit to both sides of the UNION to avoid slow queries + // and to the query itself to satisfy the user's requested limit + psStringAppend(&limitString, "\n%s", psDBGenerateLimitSQL(limit)); + psStringAppend(&query, "%s", limitString); + } + psString thisWhere = NULL; + if (whereStr) { + psStringAppend(&thisWhere, whereStr); + } + psStringAppend(&thisWhere, "\nAND warpRun.label = '%s'", label); + + if (!p_psDBRunQueryF(config->dbh, query, thisWhere, limitString, thisWhere, limitString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(limitString); + psFree(thisWhere); psFree(query); - return false; - } - psFree(limitString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + long outputLength = psArrayLength(output); + if (outputLength) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } else { + psTrace("warptool", PS_LOG_INFO, "no rows found for %s", label); + psFree(output); + continue; + } + psFree(output); + + if (limit) { + limit -= outputLength; + if (limit <= 0) { + // All done + break; + } + } + } + psFree(labelOutput); psFree(whereStr); - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); + psFree(originalQuery); return true;