Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 23500) +++ trunk/ippTools/share/Makefile.am (revision 23502) @@ -76,4 +76,6 @@ difftool_definebyquery_part2.sql \ difftool_definebyquery_temp_create.sql \ + difftool_definewarpwarp_select.sql \ + difftool_definewarpwarp_insert.sql \ difftool_donecleanup.sql \ difftool_inputskyfile.sql \ Index: trunk/ippTools/share/difftool_definewarpwarp_insert.sql =================================================================== --- trunk/ippTools/share/difftool_definewarpwarp_insert.sql (revision 23502) +++ trunk/ippTools/share/difftool_definewarpwarp_insert.sql (revision 23502) @@ -0,0 +1,16 @@ +INSERT INTO diffInputSkyfile +SELECT + %s, -- diff_id + skycell_id, + %s, -- input warp_id + 0, -- input stack_id + %s, -- template warp_id + 0, -- template stack_id + tess_id, + 0 -- diff_skyfile_id +FROM warpSkyfile AS inputWarpSkyfile +JOIN warpSkyfile AS templateWarpSkyfile USING(skycell_id, tess_id) +WHERE inputWarpSkyfile.ignored = 0 + AND templateWarpSkyfile.ignored = 0 + AND inputWarpSkyfile.warp_id = %s + AND templateWarpSkyfile.warp_id = %s Index: trunk/ippTools/share/difftool_definewarpwarp_select.sql =================================================================== --- trunk/ippTools/share/difftool_definewarpwarp_select.sql (revision 23502) +++ trunk/ippTools/share/difftool_definewarpwarp_select.sql (revision 23502) @@ -0,0 +1,25 @@ +SELECT + inputWarpRun.warp_id AS input_warp_id, + inputRawExp.exp_id AS input_exp_id, + inputWarpRun.tess_id AS tess_id, + SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id +FROM warpRun AS inputWarpRun +JOIN fakeRun AS inputFakeRun USING(fake_id) +JOIN camRun AS inputCamRun USING(cam_id) +JOIN chipRun AS inputChipRun USING(chip_id) +JOIN rawExp AS inputRawExp USING(exp_id) +-- To find exposures that haven't been diffed:%s LEFT JOIN diffRun ON diffRun.exp_id = inputRawExp.exp_id +JOIN warpRun AS templateWarpRun + ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template! + AND templateWarpRun.tess_id = inputWarpRun.tess_id -- Ensure using same tessellation +JOIN fakeRun AS templateFakeRun + ON templateFakeRun.fake_id = templateWarpRun.fake_id +JOIN camRun AS templateCamRun + ON templateCamRun.cam_id = templateFakeRun.cam_id +JOIN chipRun AS templateChipRun + ON templateChipRun.chip_id = templateCamRun.chip_id +JOIN rawExp AS templateRawExp + ON templateRawExp.exp_id = templateChipRun.exp_id + AND templateRawExp.filter = inputRawExp.filter +-- WHERE hook %s +GROUP BY input_id Index: trunk/ippTools/src/difftool.c =================================================================== --- trunk/ippTools/src/difftool.c (revision 23500) +++ trunk/ippTools/src/difftool.c (revision 23502) @@ -41,4 +41,5 @@ static bool definepoprunMode(pxConfig *config); static bool definebyqueryMode(pxConfig *config); +static bool definewarpwarpMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -79,4 +80,5 @@ MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -1006,8 +1008,15 @@ psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1018,12 +1027,17 @@ psErrorCode err = psErrorCodeLast(); switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1031,4 +1045,8 @@ psTrace("difftool", PS_LOG_INFO, "no rows found"); psFree(output); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } @@ -1039,4 +1057,7 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1059,4 +1080,7 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1072,4 +1096,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1080,4 +1107,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1088,4 +1118,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1096,4 +1129,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1104,4 +1140,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1111,4 +1150,7 @@ psFree(stackQuery); psFree(skycell_query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1143,5 +1185,8 @@ psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - return true; + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; } run->diff_id = psDBLastInsertID(config->dbh); @@ -1154,4 +1199,7 @@ psFree(skycell_query); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1165,4 +1213,7 @@ psFree(skycell_query); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1177,4 +1228,7 @@ psFree(skycell_query); psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1188,4 +1242,10 @@ psFree(skycell_query); + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + if (!diffRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); @@ -1197,4 +1257,200 @@ return true; } + +static bool definewarpwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + // Restrictions + PXOPT_COPY_S64(config->args, where, "-warp_id", "inputWarpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "inputRawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "inputRawExp.filter", "=="); + PXOPT_COPY_STR(config->args, where, "-input_label", "inputWarpRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-template_label", "templateWarpRun.label", "=="); + PXOPT_COPY_F32(config->args, where, "-timediff", + "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + PXOPT_COPY_F32(config->args, where, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<="); + + // Additional controls + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + + // Settings to apply to defined run + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psString select = pxDataGet("difftool_definewarpwarp_select.sql"); + if (!select) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = NULL; // The WHERE part of the query + + if (psListLength(where->list)) { + psString new = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new); + psFree(new); + } + psFree(where); + + if (!rerun) { + psStringAppend(&whereClause, "\n%s diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE"); + } + + if (!available) { + psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE"); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + psFree(whereClause); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, + !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun? + whereClause)) { + psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause); + psFree(select); + psFree(whereClause); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(select); + psFree(whereClause); + + psArray *results = p_psDBFetchResult(config->dbh); // Results of query + if (!results) { + psErrorCode err = psErrorCodeLast(); // Code for error + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(results)) { + psTrace("difftool", 1, "no rows found"); + psFree(results); + return true; + } + + psString insert = pxDataGet("difftool_definewarpwarp_insert.sql"); // Insertion for each new run + + psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print + long numGood = 0; // Number of good rows added + for (long i = 0; i < results->n; i++) { + psMetadata *row = results->data[i]; // Result row from query + + psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id"); + psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id"); + psS64 template_id = psMetadataLookupS64(NULL, row, "template_warp_id"); + const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + if (!exp_id || !input_id || !template_id || !tess_id) { + psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, + tess_id, exp_id, false); // Run to insert + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + run->diff_id = psDBLastInsertID(config->dbh); // Difference run identifier + + // Convert identifiers to string, for insertion into query + psString diff = NULL, input = NULL, template = NULL; // String versions of identifiers + psStringAppend(&diff, "%" PRId64, run->diff_id); + psStringAppend(&input, "%" PRId64, input_id); + psStringAppend(&template, "%" PRId64, template_id); + + if (!p_psDBRunQueryF(config->dbh, insert, diff, input, template, input, template)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(input); + psFree(template); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(input); + psFree(template); + psFree(diff); + + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psArrayAdd(list, list->n, run); + psFree(run); // Drop reference + + numGood++; + } + psFree(insert); + psFree(results); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + + if (!diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return false; + } + psFree(list); + + return true; +} + static bool pendingcleanuprunMode(pxConfig *config) @@ -1453,5 +1709,5 @@ char sqlFilename[80]; } ExportTable; - + int numExportFiles = 3; @@ -1532,11 +1788,11 @@ { unsigned int nFail; - + int numImportTables = 2; - + char tables[2] [80] = {"diffInputSkyfile", "diffSkyfile"}; PS_ASSERT_PTR_NON_NULL(config, NULL); - + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); @@ -1563,5 +1819,5 @@ psAssert (item, "entry not in input?"); psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - + switch (i) { case 0: @@ -1577,5 +1833,5 @@ } break; - + case 1: for (int i = 0; i < item->data.list->n; i++) { Index: trunk/ippTools/src/difftool.h =================================================================== --- trunk/ippTools/src/difftool.h (revision 23500) +++ trunk/ippTools/src/difftool.h (revision 23502) @@ -35,4 +35,5 @@ DIFFTOOL_MODE_DEFINEPOPRUN, DIFFTOOL_MODE_DEFINEBYQUERY, + DIFFTOOL_MODE_DEFINEWARPWARP, DIFFTOOL_MODE_PENDINGCLEANUPRUN, DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: trunk/ippTools/src/difftoolConfig.c =================================================================== --- trunk/ippTools/src/difftoolConfig.c (revision 23500) +++ trunk/ippTools/src/difftoolConfig.c (revision 23502) @@ -165,4 +165,21 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definewarpwarp + psMetadata *definewarpwarpArgs = psMetadataAlloc(); + psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-input_label", 0, "search by warp label for input", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-template_label", 0, "search by warp label for template", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); + psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -215,4 +232,5 @@ PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);