Index: branches/simmosaic_branches/ippTools/src/difftool.c =================================================================== --- branches/simmosaic_branches/ippTools/src/difftool.c (revision 24860) +++ branches/simmosaic_branches/ippTools/src/difftool.c (revision 27839) @@ -37,4 +37,5 @@ static bool todiffskyfileMode(pxConfig *config); static bool adddiffskyfileMode(pxConfig *config); +static bool advanceMode(pxConfig *config); static bool diffskyfileMode(pxConfig *config); static bool revertdiffskyfileMode(pxConfig *config); @@ -42,4 +43,5 @@ static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); +static bool definestackstackMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -50,5 +52,13 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); -static bool diffRunComplete(pxConfig *config); +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); +static bool tofullskyfileMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -76,9 +86,11 @@ MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADVANCE, advanceMode); MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); + MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK, definestackstackMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -87,4 +99,11 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); + MODECASE(DIFFTOOL_MODE_LISTRUN, listrunMode); + MODECASE(DIFFTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); + default: psAbort("invalid option (this should not happen)"); @@ -114,10 +133,14 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_S16(diff_mode, config->args, "-set_diff_mode", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -130,4 +153,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -136,5 +161,7 @@ bothways, exposure, - false + false, + diff_mode, + note ); if (!run) { @@ -167,14 +194,25 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required options - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { - // set detRun.state to state - return setdiffRunState(config, diff_id, state, false); - } - - return true; + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group","data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE diffRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true); + + psFree(query); + psFree(where); + + return result; } @@ -331,27 +369,23 @@ psString query = pxDataGet("difftool_inputskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString whereClause = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psStringCopy(""); if (psListLength(where->list)) { whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); + psStringPrepend(&whereClause, "\n WHERE "); } psFree(where); // Add condition to get only templates or only inputs + psString templateClause = psStringCopy(""); { - psString templateClause = NULL; if (template) { - psStringAppend(&templateClause, " %s", " template != 0"); + psStringAppend(&templateClause, "\n WHERE %s", " template != 0"); } else if (input) { - psStringAppend(&templateClause, " %s", " template = 0"); - } - if (templateClause) { - psStringAppend(&whereClause, "\n AND %s", templateClause); - } - psFree(templateClause); + psStringAppend(&templateClause, "\n WHERE %s", " template = 0"); + } } @@ -364,10 +398,12 @@ } - if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause, templateClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(templateClause); psFree(whereClause); psFree(query); return false; } + psFree(templateClause); psFree(whereClause); psFree(query); @@ -421,5 +457,5 @@ psString query = pxDataGet("difftool_todiffskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -555,12 +591,4 @@ } - if (!diffRunComplete(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -569,4 +597,95 @@ } + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereString); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + psFree(whereString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + // set diffRun.state to 'full' + if (!setdiffRunState(config, diff_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -581,17 +700,40 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); - PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "=="); - + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffSkyfile"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); psString query = pxDataGet("difftool_skyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -601,7 +743,20 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; } psFree(where); + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + // treat limit == 0 as "no limit" if (limit) { @@ -659,21 +814,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "diffRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - // Update state to 'new' + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + { - psString query = pxDataGet("difftool_revertdiffskyfile_update.sql"); + psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("difftool_revertdiffskyfile_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -682,4 +839,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -688,50 +846,21 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } return false; } psFree(query); + psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh)); } - // Delete product - { - psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); - } - psFree(where); - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } return true; @@ -749,14 +878,47 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { + if (magicked) { + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for diff_id %"PRId64, diff_id); return false; + } + } + else { + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } } return true; } + + +#ifdef notdef +static bool setdiffRunStateByLabel(pxConfig *config, const char *label, const char *state) { + PS_ASSERT_PTR_NON_NULL(state,false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh,query,state,label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return(false); + } + + return true; +} +#endif // Generate a single populated run @@ -766,5 +928,8 @@ const char *tess_id, // Tessellation identifier const char *label, // label + const char *data_group, // data_group + const char *dist_group, // dist_group const char *reduction, // reduction + const char *note, // note psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none @@ -787,4 +952,18 @@ return false; } + psS16 diff_mode = IPP_DIFF_MODE_UNDEFINED; + if ((input_warp_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_WARP; + } + else if ((input_warp_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_WARP_STACK; + } + else if ((input_stack_id != PS_MAX_S64) && (template_warp_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_WARP; + } + else if ((input_stack_id != PS_MAX_S64) && (template_stack_id != PS_MAX_S64)) { + diff_mode = IPP_DIFF_MODE_STACK_STACK; + } + // default @@ -797,4 +976,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -803,5 +984,7 @@ false, false, - 0 // magicked + 0, // magicked + diff_mode, // diff_mode + note ); @@ -877,4 +1060,6 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); @@ -883,4 +1068,5 @@ PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); if (template_stack_id && template_warp_id) { @@ -905,6 +1091,6 @@ psArray *list = psArrayAllocEmpty(16); // List of runs, to print - - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + // Populated Run will generate the diff_mode value + if (!populatedrun(list, workdir, skycell_id, tess_id, label, data_group ? data_group : label, dist_group, reduction, note, input_warp_id ? input_warp_id : PS_MAX_S64, input_stack_id ? input_stack_id : PS_MAX_S64, @@ -933,20 +1119,31 @@ psMetadata *expWhere = psMetadataAlloc(); - psMetadata *warpWhere = psMetadataAlloc(); + psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp + psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp psMetadata *stackWhere = psMetadataAlloc(); PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "=="); - PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "=="); - PXOPT_COPY_F32(config->args, warpWhere, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, expWhere, "-comment", "comment", "LIKE"); + PXOPT_COPY_S64(config->args, warp1Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_F32(config->args, warp2Where, "-good_frac", "warpSkyfile.good_frac", ">="); PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); @@ -958,9 +1155,10 @@ psString query = pxDataGet("difftool_definewarpstack_part1.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString warpQuery = NULL; + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString warp1Query = NULL; + psString warp2Query = NULL; psString stackQuery = NULL; psString expQuery = NULL; @@ -974,17 +1172,20 @@ } psFree(expWhere); - if (psListLength(warpWhere->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); - psStringAppend(&warpQuery, "\n AND %s", whereClause); + if (psListLength(warp1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL); + psStringAppend(&warp1Query, "\n AND %s", whereClause); psFree(whereClause); } else { - warpQuery = psStringCopy("\n"); - } - psFree(warpWhere); - - if (!available) { - // diff what's available, even if warp run has some faults and is incomplete - psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); - } + warp1Query = psStringCopy("\n"); + } + psFree(warp1Where); + if (psListLength(warp2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL); + psStringAppend(&warp2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + warp2Query = psStringCopy("\n"); + } + psFree(warp2Where); // don't queue for exposures that have already been diff'd unless requested @@ -1005,5 +1206,5 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery); if (!psDBTransaction(config->dbh)) { @@ -1012,5 +1213,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) { + if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1072,4 +1273,5 @@ return false; } + psFree(warp1Query); psFree(query); query = NULL; @@ -1103,5 +1305,5 @@ if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1114,5 +1316,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1125,5 +1327,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1136,5 +1338,5 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1144,8 +1346,9 @@ return false; } + psString filter = psMetadataLookupStr(&mdok, row, "filter"); if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "filter not found"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1155,7 +1358,23 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warpQuery)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + + psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warp data_group not found"); + psFree(warp2Query); + psFree(stackQuery); + psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (warp_data_group) { + data_group = warp_data_group; + } + + if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1168,11 +1387,11 @@ if (num == 0) { - psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id); continue; } if (!available && (num != skycell_count)) { - psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" - PRId64 " need %" PRId64, num, skycell_count, warp_id); + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64 + " but need %" PRId64, num, warp_id, skycell_count); continue; } @@ -1184,4 +1403,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -1190,5 +1411,7 @@ false, // bothways true, // exposure - 0 // magicked + 0, // magicked + IPP_DIFF_MODE_WARP_STACK, + note ); @@ -1206,5 +1429,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1220,5 +1443,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1235,5 +1458,5 @@ psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1249,5 +1472,5 @@ } psFree(output); - psFree(warpQuery); + psFree(warp2Query); psFree(stackQuery); psFree(skycell_query); @@ -1259,5 +1482,5 @@ } - if (!diffRunPrintObjects(stdout, list, !simple)) { + if (numGood && !diffRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -1288,4 +1511,35 @@ "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + // other where restrictions: + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_begin", "inputRawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_end", "inputRawExp.dateobs", "<="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_min", "inputRawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_max", "inputRawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_min", "inputRawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_max", "inputRawExp.decl", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_min", "inputRawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_max", "inputRawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_min", "inputRawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_max", "inputRawExp.airmass", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_min", "inputRawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_max", "inputRawExp.exp_time", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_min", "inputRawExp.bg", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_max", "inputRawExp.bg", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_min", "inputRawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_max", "inputRawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_min", "inputRawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_max", "inputRawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_min", "inputRawExp.alt", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_max", "inputRawExp.alt", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-az_min", "inputRawExp.az", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-az_max", "inputRawExp.az", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_min", "inputRawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_max", "inputRawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_min", "inputRawExp.posang", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_max", "inputRawExp.posang", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_min", "inputRawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_max", "inputRawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, selectWhere, "-comment", "inputRawExp.comment", "LIKE"); + // Haversine formula for great circle distance PXOPT_COPY_F32(config->args, selectWhere, "-distance", @@ -1306,11 +1560,14 @@ // Additional controls PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // Settings to apply to defined run - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); // option + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); // option + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -1327,5 +1584,5 @@ psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL if (!tempCreate) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1350,5 +1607,5 @@ psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table if (!tempInsert) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1382,5 +1639,5 @@ psString select = pxDataGet("difftool_definewarpwarp_select.sql"); if (!select) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(selectWhere); psFree(insertWhere); @@ -1395,18 +1652,11 @@ if (psListLength(selectWhere->list)) { psString new = psDBGenerateWhereConditionSQL(selectWhere, NULL); - psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new); + psStringAppend(&whereClause, "\nAND %s", new); psFree(new); } psFree(selectWhere); - if (!available) { - psStringAppend(&whereClause, - "\n%s inputWarpRun.state = 'full'" - "AND templateWarpRun.state = 'full'", - whereClause ? "AND" : "WHERE"); - } - if (!rerun) { - psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE"); + psStringAppend(&whereClause, "\nAND diffs.diff_id IS NULL"); } @@ -1485,4 +1735,9 @@ const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); + if (input_data_group) { + data_group = input_data_group; + } + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); @@ -1495,7 +1750,20 @@ return false; } - - diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, true, true, false); // Run to insert + + diffRunRow *run = diffRunRowAlloc(0, + "reg", + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + true, // bothways + true, // exposure + false, // magicked + IPP_DIFF_MODE_WARP_WARP, + note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1569,4 +1837,303 @@ } +static bool definestackstackMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *stack1Where = psMetadataAlloc(); + psMetadata *stack2Where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, stack1Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","=="); + PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","=="); + PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + PXOPT_COPY_F32(config->args, stack2Where, "-good_frac", "stackSumSkyfile.good_frac", ">="); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(newTemplates,config->args,"-new-templates", false); + + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + if (!(label)) { + PXOPT_LOOKUP_STR(label,config->args, "-input_label",true,false); + } + + // Organize the config information into queries. + psString stack1Query = NULL; + psString stack2Query = NULL; + + if (psListLength(stack1Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack1Where, NULL); + psStringAppend(&stack1Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack1Query = psStringCopy("\n"); + } + psFree(stack1Where); + + if (psListLength(stack2Where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(stack2Where, NULL); + psStringAppend(&stack2Query, "\n AND %s", whereClause); + psFree(whereClause); + } else { + stack2Query = psStringCopy("\n"); + } + psFree(stack2Where); + + // don't queue for stacks that have already been diffed unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\n AND diffExp.diff_id IS NULL"); + } else { + diffQuery = psStringCopy("\n"); + } + + // find the distinct set of data_groups and filters + psString query = pxDataGet("difftool_definestackstack_part0.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psTrace("difftool",1,query,stack2Query,stack1Query,diffQuery,stack1Query); +/* exit(10); */ + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, stack1Query, diffQuery, stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + query = pxDataGet("difftool_definestackstack_part1.sql"); + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRunTemp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Output row from query + bool mdok; // Status of MD lookup + + psString tess_id = psMetadataLookupStr(&mdok,row,"INPUT_tess_id"); + psString this_data_group = psMetadataLookupStr(&mdok,row,"INPUT_data_group"); + psString this_dist_group = psMetadataLookupStr(&mdok,row,"INPUT_dist_group"); + psString this_label = psMetadataLookupStr(&mdok,row,"INPUT_label"); + + psString this_stack1Query = psStringCopy(stack1Query); + + psString thisWhere = psDBGenerateWhereConditionSQL(row,NULL); + psStringSubstitute(&thisWhere,"stackRun.","INPUT_"); + psStringAppend(&this_stack1Query,"\n AND %s", thisWhere); + psFree(thisWhere); + + psTrace("difftool",1, query,stack2Query,this_stack1Query,diffQuery,this_stack1Query); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query, stack2Query, this_stack1Query, diffQuery, this_stack1Query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(this_stack1Query); + + psArray *diff_id_output = p_psDBFetchResult(config->dbh); + if (!diff_id_output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(diff_id_output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(diff_id_output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label ? label : this_label, + data_group ? data_group : this_data_group, + dist_group ? dist_group : this_dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + false, // bothways + false, // exposure + 0, // magicked + IPP_DIFF_MODE_STACK_STACK, // diff_mode + note + ); + // Commit to database + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } +/* diffRunPrintObject(stdout,run,1); */ + run->diff_id = psDBLastInsertID(config->dbh); + for (long j = 0; j < diff_id_output->n; j++) { + psMetadata *input_row = diff_id_output->data[j]; // Output row from query + bool mdok; + psString in_skycell_id = psMetadataLookupStr(&mdok,input_row,"skycell_id"); + psS64 in_input_stack_id = psMetadataLookupS64(&mdok,input_row,"stack_id"); + psS64 in_template_stack_id = psMetadataLookupS64(&mdok,input_row,"max_stack_id"); + psString in_tess_id = psMetadataLookupStr(&mdok,input_row,"tess_id"); + psTrace("difftool",1,"%s %" PRId64 " %" PRId64 " %s\n", + in_skycell_id, in_input_stack_id, in_template_stack_id, in_tess_id); + diffInputSkyfileRow *input = diffInputSkyfileRowAlloc( + run->diff_id, // ID + in_skycell_id, + PS_MAX_S64, // warp1_id -> NULL + in_input_stack_id, // stack1 + PS_MAX_S64, // warp2_id -> NULL + in_template_stack_id, // stack2 + in_tess_id, + 0 // diff_skyfile_id + ); + // Commit to database the input + if (!diffInputSkyfileInsertObject(config->dbh, input)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + +/* diffInputSkyfilePrintObject(stdout,input,1); */ + psFree(input); + } + + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, run->diff_id); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(run); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psArrayAdd(list, list->n, run); + psFree(run); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + + psFree(query); + psFree(stack1Query); + psFree(stack2Query); + psFree(diffQuery); + psFree(output); + psFree(list); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + return(true); +} static bool pendingcleanuprunMode(pxConfig *config) @@ -1582,12 +2149,20 @@ psString query = pxDataGet("difftool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // pxDataGet/psStringSubstitute workaround + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereClause, " AND "); + psStringSubstitute(&query,whereClause,"@INNERCONSTRAINT@"); psFree(whereClause); + } + else { + psStringSubstitute(&query,NULL,"@INNERCONSTRAINT@"); } psFree(where); @@ -1599,5 +2174,7 @@ psFree(limitString); } - + // fprintf(stderr,"%s",query); + + //fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1647,5 +2224,5 @@ psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1707,5 +2284,5 @@ psString query = pxDataGet("difftool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1762,4 +2339,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) { @@ -1773,49 +2351,85 @@ } -static bool diffRunComplete(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // look for completed diffRuns - psString query = pxDataGet("difftool_completed_runs.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("difftool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); - - // set diffRun.state to 'stop' - if (!setdiffRunState(config, diff_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, - diff_id); - psFree(output); - return false; - } - } - - return true; -} +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) { + PS_ASSERT_PTR_NON_NULL(config, false); + + // diff_id, skycell_id are required + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + + psS64 magicked = 0; + if (!strcmp(data_state, "full")) { + PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false); + magicked = set_magicked; + } + + psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (magicked != 0 && !strcmp(data_state, "full")) { + psStringAppend(&set_magicked_skyfile, "\n , diffSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n , diffRun.magicked = %" PRId64, magicked); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)"); + psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)"); + } + + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_skyfile); + return(false); + } + psFree(set_magicked_skyfile); + psFree(query); + + query = pxDataGet("difftool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); + return(false); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(set_magicked_run); + return(false); + } + psFree(set_magicked_run); + + return(true); +} + +static bool tocleanedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "cleaned","goto_cleaned"); +} +static bool topurgedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +static bool tofullskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "full", "update"); +} + bool exportrunMode(pxConfig *config) @@ -1841,4 +2455,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); @@ -1853,5 +2471,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1932,7 +2550,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "diffRun"); psAssert (item, "entry not in input?"); @@ -1984,2 +2608,148 @@ return true; } + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "diffRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "LIKE"); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "LIKE"); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffRun"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); + psString query = pxDataGet("difftool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); + } else if (!all) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + if (template) { + psStringAppend(&query, " ORDER BY rawTemplate.exp_id, diff_id DESC"); + } else { + psStringAppend(&query, " ORDER BY rawInput.exp_id, diff_id DESC"); + } + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("difftool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , diffRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +}