Index: branches/bills_081204/ippTools/src/difftool.c =================================================================== --- branches/bills_081204/ippTools/src/difftool.c (revision 20903) +++ branches/bills_081204/ippTools/src/difftool.c (revision 20954) @@ -121,11 +121,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - exp_id, - tess_id + tess_id, + exp_id ); if (!run) { @@ -234,9 +234,9 @@ if (!diffInputSkyfileInsert(config->dbh, diff_id, + skycell_id, warp1 ? warp1 : PS_MAX_S64, // defined or NULL + stack1 ? stack1 : PS_MAX_S64, // defined or NULL warp2 ? warp2 : PS_MAX_S64, // defined or NULL - stack1 ? stack1 : PS_MAX_S64, // defined or NULL stack2 ? stack2 : PS_MAX_S64, // defined or NULL - skycell_id, tess_id )) { @@ -304,5 +304,5 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); @@ -531,5 +531,7 @@ hostname, good_frac, - code + code, + 0, // diff_image_id + 0 // magic_ds_id )) { if (!psDBRollback(config->dbh)) { @@ -795,11 +797,11 @@ 0, // ID "reg", // state + workdir, label, reduction, - workdir, NULL, // dvodb registered, - exp_id, - tess_id + tess_id, + exp_id ); @@ -826,9 +828,9 @@ if (!diffInputSkyfileInsert(config->dbh, run->diff_id, + skycell_id, input_warp_id, + input_stack_id, template_warp_id, - input_stack_id, template_stack_id, - skycell_id, tess_id )) { @@ -936,7 +938,9 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *expWhere = psMetadataAlloc(); psMetadata *warpWhere = psMetadataAlloc(); psMetadata *stackWhere = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "=="); PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpRun.skycell_id", "=="); @@ -953,7 +957,9 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); // find all things to queue - psString query = pxDataGet("difftool_definebyquery.sql"); + psString query = pxDataGet("difftool_definebyquery_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -961,15 +967,16 @@ } - if (newTemplates) { - // Warps that haven't been diffed OR warps that can take a newer template in the diff - psStringAppend(&query, " WHERE (diff_id IS NULL OR best_stack_id > current_stack_id)"); - } else { - // Only warps that haven't been diffed - psStringAppend(&query, " WHERE diff_id IS NULL"); - } - psString warpQuery = NULL; psString stackQuery = NULL; - + psString expQuery = NULL; + + if (psListLength(expWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL); + psStringAppend(&expQuery, "\n AND %s", whereClause); + psFree(whereClause); + } else { + expQuery = psStringCopy("\n"); + } + psFree(expWhere); if (psListLength(warpWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL); @@ -977,11 +984,22 @@ psFree(whereClause); } else { - warpQuery = psStringCopy(""); + warpQuery = psStringCopy("\n"); } psFree(warpWhere); + + if (!available) { + // diff what's available, even if warp run has some faults and is incomplete + psStringAppend(&warpQuery, " AND warpRun.state = 'full'"); + } + + // don't queue for exposures that have already been diff'd unless requested + psString diffQuery = NULL; + if (! (reRun || newTemplates) ) { + psStringAppend(&diffQuery, "\nAND diff_id IS NULL"); + } if (psListLength(stackWhere->list)) { psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL); - psStringAppend(&stackQuery, "\n AND %s", whereClause); + psStringAppend(&stackQuery, "\nAND %s", whereClause); psFree(whereClause); } else { @@ -990,7 +1008,8 @@ psFree(stackWhere); - psTrace("difftool", 1, query, warpQuery, stackQuery); - - if (!p_psDBRunQuery(config->dbh, query, warpQuery, stackQuery)) { + psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + + + if (!p_psDBRunQuery(config->dbh, query, warpQuery, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1019,45 +1038,140 @@ } + // create temporary table + query = pxDataGet("difftool_definebyquery_temp_create.sql"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + query = NULL; + + psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql"); + psArray *list = psArrayAllocEmpty(16); // List of runs, to print long numGood = 0; // Number of good rows added + psS64 last_exp_id = 0; for (long i = 0; i < output->n; i++) { psMetadata *row = output->data[i]; // Output row from query - - // Selected parameters bool mdok; // Status of MD lookup - const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); - if (!mdok || !skycell_id) { - psWarning("skycell_id not found --- ignoring row %ld", i); + + // Take the first warp for each exposure. + // The list is sorted by exposure id and warp_id and the warps are in descending + // order. + psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + return false; + } + if (exp_id == last_exp_id) { continue; } - const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); - if (!mdok || !tess_id) { - psWarning("tess_id not found --- ignoring row %ld", i); - continue; - } - psS64 stack_id = psMetadataLookupS64(&mdok, row, "best_stack_id"); - if (!mdok) { - psWarning("stack_id not found --- ignoring row %ld", i); - continue; + last_exp_id = exp_id; + + // clear temporary table + if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; } psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); if (!mdok) { - psWarning("warp_id not found --- ignoring row %ld", i); + psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "skycell_count not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + if (!p_psDBRunQuery(config->dbh, skycell_query, warp_id, warpQuery, stackQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + return false; + } + psS64 num = psDBAffectedRows(config->dbh); + + if (num == 0) { + psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id); continue; } - psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); - if (!mdok) { - psWarning("exp_id not found --- ignoring row %ld", i); + + if (!available && (num != skycell_count)) { + psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" + PRId64 " need %" PRId64, num, skycell_count, warp_id); continue; } - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, warp_id, - PS_MAX_S64, PS_MAX_S64, stack_id, exp_id, config)) { - psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id, - warp_id, stack_id); - psErrorClear(); - continue; - } - + // ok we've got one create the diffRun + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label, + reduction, + NULL, // dvodb + registered, + tess_id, + exp_id + ); + + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return true; + } + run->diff_id = psDBLastInsertID(config->dbh); + + psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + psFree(query); + query = NULL; + psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff"); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + + if (!setdiffRunState(config, run->diff_id, "new")) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(warpQuery); + psFree(stackQuery); + psFree(skycell_query); + psFree(query); + return false; + } + + psArrayAdd(list, list->n, run); numGood++; }