Index: trunk/ippTools/src/addtool.c =================================================================== --- trunk/ippTools/src/addtool.c (revision 28154) +++ trunk/ippTools/src/addtool.c (revision 28181) @@ -40,4 +40,15 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); +static bool addminidvodbrunMode(pxConfig *config); +static bool updateminidvodbrunMode(pxConfig *config); +static bool listminidvodbrunMode(pxConfig *config); +static bool flipminidvodbrunMode(pxConfig *config); +static bool checkminidvodbrunaddrunMode(pxConfig *config); +static bool addminidvodbprocessedMode(pxConfig *config); +static bool listminidvodbprocessedMode(pxConfig *config); +static bool revertminidvodbprocessedMode(pxConfig *config); +static bool updateminidvodbprocessedMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -69,4 +80,14 @@ MODECASE(ADDTOOL_MODE_MASKED, maskedMode); MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_ADDMINIDVODBRUN, addminidvodbrunMode); + MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBRUN, updateminidvodbrunMode); + MODECASE(ADDTOOL_MODE_LISTMINIDVODBRUN, listminidvodbrunMode); + MODECASE(ADDTOOL_MODE_FLIPMINIDVODBRUN, flipminidvodbrunMode); + MODECASE(ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, checkminidvodbrunaddrunMode); + MODECASE(ADDTOOL_MODE_ADDMINIDVODBPROCESSED,addminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_LISTMINIDVODBPROCESSED,listminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_REVERTMINIDVODBPROCESSED,revertminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED,updateminidvodbprocessedMode); + default: psAbort("invalid option (this should not happen)"); @@ -102,4 +123,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + if (!psListLength(where->list)) { psFree(where); @@ -114,8 +136,13 @@ PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(minidvodb, config->args, "-set_minidvodb", false); PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the cam_id of all the exposures that we want to queue up. @@ -245,8 +272,11 @@ dvodb ? dvodb : row->dvodb, note ? note : NULL, - image_only + image_only, + minidvodb, + minidvodb_group, + minidvodb_name )) { if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error sfg"); } psError(PS_ERR_UNKNOWN, false, @@ -291,5 +321,5 @@ // pxUpdateRun gets parameters from config->args and runs the update query - bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", + bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", "addProcessedExp", false); @@ -374,4 +404,5 @@ PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + PXOPT_LOOKUP_STR(dvodb_path, config->args, "-dvodb_path", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -399,4 +430,5 @@ return false; } + //psError(PS_ERR_UNKNOWN, false, "database error %s", query); psFree(query); @@ -424,4 +456,5 @@ dtime_addstar, path_base, + dvodb_path, fault ); @@ -522,5 +555,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); psFree(query); return false; @@ -733,2 +766,841 @@ return true; } + +static bool addminidvodbrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); + + //optional + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString minidvodbpath = "NULL"; + + // I don't know how to get around the complaints of minidvodb_path can't be null. this 'fixes' it, but someone smarter can fix it properly. + if (minidvodb_path) { + minidvodbpath = minidvodb_path; + } + + if (!minidvodbRunInsert(config->dbh, + 0, // job_id + minidvodb_name, + minidvodb_group, + minidvodbpath, + mergedvodb_path, + "new", + 0 + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 minidvodb_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", minidvodb_id); + + + if (!minidvodb_name) { + psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); + } + + if (!minidvodb_path) { + psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); + } + + psString query = NULL; + + psStringAppend(&query, "UPDATE minidvodbRun SET minidvodb_path = '%s', minidvodb_name = '%s' where minidvodb_id = %" PRIu64";", minidvodb_path, minidvodb_name, minidvodb_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updateminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); + + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE minidvodbRun SET "); + int cnt = 0; + psString comma = ","; + if (minidvodb_name) { + psStringAppend(&query, " minidvodb_name = '%s'", minidvodb_name); + cnt++; + } + + if (minidvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " minidvodb_path = '%s'", minidvodb_path); + cnt++; + } + + if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + if (minidvodb_group) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " minidvodb_group = '%s'", minidvodb_group); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; + +} + +static bool flipminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group",true, false); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); + +//this flips the new - > active +// and the active - > waiting in one action + + +// the first query looks to find things that are new and where all the fields are filled (ie, ready to be flipped to active) + +// start a transaction eraly so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + + psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&firstquery, " AND %s", firstwhereClause); + + + + if (!p_psDBRunQuery(config->dbh, firstquery)) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(firstquery); + + //we don't care what the stuff is that is found, just that there is stuff. This is a check to see that there is something in the 'new' state, before flipping (so that if there is nothing in new, it won't flip the active to waiting. the flipminidvo is just to make it easy to flip from new -> active. + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + + //ok, there's something, so flip active -> waiting + + psString query = psStringCopy("UPDATE minidvodbRun SET state = 'waiting' WHERE state = 'active' "); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + + //now flip new -> active + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); + if (!p_psDBRunQuery(config->dbh, query2)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(query); + return false; + } + + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(query); + return false; + } + psFree(query2); + psFree(where); + + return true; + +} + + +static bool checkminidvodbrunaddrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + //this checks to see if a minidvod_group/name is has completed addRun processing + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(all_addrun_states, config->args, "-all_addrun_states", false); + //this doesn't care what state the addRun is in (useful for counting addRuns) + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_checkminidvodbaddrun.sql"); + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (!all_addrun_states) { + psStringAppend(&query, " AND (cnt2 = cnt) "); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + + +static bool listminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_find_minidvodbrun.sql"); + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool addminidvodbprocessedMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); + + // optional + PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); + PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + //generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + psString query = pxDataGet("addtool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + minidvodbRunRow *pendingRow = minidvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( + pendingRow->minidvodb_id, + merge_order, + dtime_resort, + dtime_relphot, + dtime_merge, + epoch, + mergedvodb_path, + fault + ); + + if (!minidvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + + + + //this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); + printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + + + +// since there is only one exp per 'new' set mindvodbRun.state = 'merged' + + psString query2 = NULL ; + psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + return true; +} + + + +static bool listminidvodbprocessedMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_find_minidvodbprocessed.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND minidvodbProcessed.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND minidvodbProcessed.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault = 0"); + } + psFree(where); + + // order by epoch + psStringAppend(&query, " ORDER BY minidvodb_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error "); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "minidvodbProcessed", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + + + +static bool revertminidvodbprocessedMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "addRun.minidvodb_group", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_minidvodbruns.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + { + psString query = pxDataGet("addtool_revertminidvodbprocessed.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + + +static bool updateminidvodbprocessedMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); + PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); + PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE minidvodbProcessed JOIN minidvodbRun USING (minidvodb_id) SET "); + int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + if (merge_order) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " merge_order = %" PRId64, merge_order); + cnt++; + } + + if (dtime_relphot) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_relphot = %f", dtime_relphot); + cnt++; + } + + if (dtime_resort) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_resort = %f", dtime_resort); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + Index: trunk/ippTools/src/addtool.h =================================================================== --- trunk/ippTools/src/addtool.h (revision 28154) +++ trunk/ippTools/src/addtool.h (revision 28181) @@ -39,5 +39,14 @@ ADDTOOL_MODE_DONECLEANUP, ADDTOOL_MODE_EXPORTRUN, - ADDTOOL_MODE_IMPORTRUN + ADDTOOL_MODE_IMPORTRUN, + ADDTOOL_MODE_ADDMINIDVODBRUN, + ADDTOOL_MODE_UPDATEMINIDVODBRUN, + ADDTOOL_MODE_LISTMINIDVODBRUN, + ADDTOOL_MODE_FLIPMINIDVODBRUN, + ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, + ADDTOOL_MODE_ADDMINIDVODBPROCESSED, + ADDTOOL_MODE_LISTMINIDVODBPROCESSED, + ADDTOOL_MODE_REVERTMINIDVODBPROCESSED, + ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED } addtoolMode; Index: trunk/ippTools/src/addtoolConfig.c =================================================================== --- trunk/ippTools/src/addtoolConfig.c (revision 28154) +++ trunk/ippTools/src/addtoolConfig.c (revision 28181) @@ -63,5 +63,7 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb", 0, "use minidvodb", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_bname", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); @@ -84,4 +86,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -98,4 +101,5 @@ psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-dvodb_path", 0, "define base output location", NULL); psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -142,4 +146,79 @@ psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + // -addminidvodbruns + psMetadata *addminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -updateminidvodbruns + psMetadata *updateminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddU64(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id ", 0); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -listminidvodbrunArgs + psMetadata *listminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb.minidvodb_group", NULL); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + //psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-finished_addrun", 0, "limit to minidvodbs with completed addRuns (none in new state)", false); + psMetadata *flipminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddStr(flipminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "for the supplied minidvodb_group (required): flip the current 'new' to 'active', the current 'active' to 'waiting'", NULL); + + psMetadata *checkminidvodbrunaddrunArgs = psMetadataAlloc(); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "for the supplied minidvodb_group (required): check if the addRun stage is complete (all in addRun.state = full) ", NULL); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "for the supplied minidvodb_name: check if the addRun stage is complete (all in addRun.state = full) ", NULL); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-state", 0, "limit by minidvodbRun state ", NULL); + psMetadataAddU64(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-all_addrun_states", 0, "list all minidvodbRun.minidvodb_names, not just ones that have complete addRuns", false); + + psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); + psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); + psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); + psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + psMetadata *listminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); + psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb.minidvodb_group", NULL); + psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by addRun.minidvodb_group", NULL); + + psMetadata *updateminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); + psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -154,5 +233,16 @@ PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); - PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-addminidvodbrun", "create minidvodbs ", ADDTOOL_MODE_ADDMINIDVODBRUN, addminidvodbrunArgs); + PXOPT_ADD_MODE("-updateminidvodbrun", "change minidvodb properties", ADDTOOL_MODE_UPDATEMINIDVODBRUN, updateminidvodbrunArgs); + PXOPT_ADD_MODE("-listminidvodbrun", "list minidvodbs", ADDTOOL_MODE_LISTMINIDVODBRUN, listminidvodbrunArgs); + PXOPT_ADD_MODE("-flipminidvodbrun", "flip minidvodbs", ADDTOOL_MODE_FLIPMINIDVODBRUN, flipminidvodbrunArgs); + PXOPT_ADD_MODE("-checkminidvodbrunaddrun", "check minidvodbs to see if addRuns are completed", ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, checkminidvodbrunaddrunArgs); + PXOPT_ADD_MODE("-addminidvodbprocessed","add a processed minidvodb", ADDTOOL_MODE_ADDMINIDVODBPROCESSED, addminidvodbprocessedArgs); + PXOPT_ADD_MODE("-listminidvodbprocessed","list processed minidvodbs", ADDTOOL_MODE_LISTMINIDVODBPROCESSED, listminidvodbprocessedArgs); + PXOPT_ADD_MODE("-revertminidvodbprocessed","revert processed minidvobs", ADDTOOL_MODE_REVERTMINIDVODBPROCESSED, revertminidvodbprocessedArgs); + PXOPT_ADD_MODE("-updateminidvodbprocessed","change processed minidvodb properties",ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED, updateminidvodbprocessedArgs); + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: trunk/ippTools/src/flatcorr.c =================================================================== --- trunk/ippTools/src/flatcorr.c (revision 28154) +++ trunk/ippTools/src/flatcorr.c (revision 28181) @@ -671,5 +671,8 @@ row->dvodb, NULL, // note is not propagated - 0)) { + 0, + 0, //The minidvodb stuff is off + NULL, + NULL)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: trunk/ippTools/src/pstamptool.h =================================================================== --- trunk/ippTools/src/pstamptool.h (revision 28154) +++ trunk/ippTools/src/pstamptool.h (revision 28181) @@ -22,4 +22,6 @@ #include "pxtools.h" + +#define PSTAMP_FIRST_ERROR_CODE 0 // First error code value to consider typedef enum { Index: trunk/ippTools/src/pxadd.c =================================================================== --- trunk/ippTools/src/pxadd.c (revision 28154) +++ trunk/ippTools/src/pxadd.c (revision 28181) @@ -130,5 +130,8 @@ char *dvodb, char *note, - bool image_only) + bool image_only, + bool minidvodb, + char *minidvodb_group, + char *minidvodb_name) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -158,7 +161,10 @@ note ? note : "NULL", image_only, + minidvodb, + minidvodb_group, + minidvodb_name, (long long) cam_id )) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error %s", query); return false; } Index: trunk/ippTools/src/pxadd.h =================================================================== --- trunk/ippTools/src/pxadd.h (revision 28154) +++ trunk/ippTools/src/pxadd.h (revision 28181) @@ -38,5 +38,8 @@ char *dvodb, char *note, - bool image_only); + bool image_only, + bool minidvodb, + char *minidvodb_group, + char *minidvodb_name); #endif // PXADD_H