Index: anches/eam_branches/ipp-20230313/ippTools/share/xcfftool_defineforstacks.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_defineforstacks.sql (revision 42590) +++ (revision ) @@ -1,14 +1,0 @@ -SELECT - xccalRun.xccal_id, - xccalResult.path_base, - xccalRun.data_group, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter -FROM xccalRun - JOIN xccalResult USING(xccal_id) - JOIN stackRun USING(stack_id) - JOIN skycell USING(tess_id, skycell_id) - -- join hook %s -WHERE - xccalRun.state = 'full' AND xccalResult.quality = 0 Index: anches/eam_branches/ipp-20230313/ippTools/share/xcfftool_defineforstacks_select_warps.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_defineforstacks_select_warps.sql (revision 42590) +++ (revision ) @@ -1,5 +1,0 @@ -SELECT warp_id -FROM stackInputSkyfile - JOIN stackRun USING (stack_id) - JOIN xccalRun USING (stack_id) -WHERE xccal_id = '%d' Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql (revision 42591) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql (revision 42591) @@ -0,0 +1,7 @@ +UPDATE xcForceRun +JOIN %s.xccalRun USING(xccal_id) +JOIN %s.xccalResult USING (xccal_id) +JOIN %s.xcstackInputSkyfile USING (xcstack_id) -- exposes the stack_id +JOIN %s.xcstackCamera USING (camera_id) -- select the camera matching this db +JOIN stackRun USING (stack_id) +JOIN skycell USING(tess_id, skycell_id)"); Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c (revision 42590) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c (revision 42591) @@ -105,5 +105,6 @@ } - +// NOTE: if I denormalize and populate (tess_id, skycell_id) in xcForceRun the I do not need +// to join back to xcstackRun to get that info static bool definebyqueryMode(pxConfig *config) { @@ -116,7 +117,8 @@ PXOPT_LOOKUP_STR(xcamera, config->args, "-xcamera", true, false); - // get this camera_id from the database: - psS64 xcamera_id = getCameraIDfromName(config, xcamera); - if (!xcamera_id) { + // get camera_id for the working database: + // XXX need to extract the camera name (-dbname) from the config + psS64 camera_id = getCameraIDfromName(config, config->dbh->camera); + if (!camera_id) { psError(PXTOOLS_ERR_SYS, false, "failed to find xcamera in database"); return false; @@ -144,7 +146,7 @@ // XXX do these need to specify stackRun? can the filters be different between stackRun and xcstackRun? - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); - pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id", "xcstackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter", "xcstackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, xccalWhereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -164,4 +166,5 @@ } PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + // XXX do I need these or not? the warp skyfiles need to match the stack skycells PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "rawExp.filter", "LIKE"); @@ -188,9 +191,10 @@ psString joinHook = NULL; if (!rerun) { - psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = xccalRun.xccal_id"); + psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id", xcamera); psStringAppend(&joinHook, "\n %s\nAND xcForceRun.label = '%s'", where, label); psStringAppend(&select, "\nAND ff_id IS NULL"); } + // XXX need to add the xcamera name here to match definebyquery.sql if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -297,5 +301,5 @@ } - // create a staticskyRun + // create an xcForceRun if (!xcForceRunInsert(config->dbh, 0x0, // ff_id @@ -320,5 +324,5 @@ } - psS64 ff_id = psDBLastInsertID(config->dbh); + psS64 xcff_id = psDBLastInsertID(config->dbh); for (int j = 0; j < warpOutput->n; j++) { @@ -326,7 +330,5 @@ psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); - if (!xcForceInputInsert(config->dbh, - ff_id, - warp_id) + if (!xcForceInputInsert(config->dbh, ff_id, warp_id, xcamera_id) ) { if (!psDBRollback(config->dbh)) { @@ -353,216 +355,4 @@ } -static bool defineforstacksMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); - - // optional - PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); - PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); - PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - - PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); - - psMetadata *xccalWhereMD = psMetadataAlloc(); - pxAddLabelSearchArgs(config, xccalWhereMD, "-select_xccal_label", "xccalRun.label", "LIKE"); - pxAddLabelSearchArgs(config, xccalWhereMD, "-select_xccal_data_group", "xccalRun.data_group", "LIKE"); - PXOPT_COPY_S64(config->args, xccalWhereMD, "-select_xccal_id", "xccalRun.xccal_id", "=="); - - if (!psListLength(xccalWhereMD->list)) { - psError(PXTOOLS_ERR_CONFIG, false, "xccal search parameters are required"); - psFree(xccalWhereMD); - return false; - } - - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); - pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); - if (!pxskycellAddWhere(config, xccalWhereMD)) { - psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); - psFree(xccalWhereMD); - return false; - } - - PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - - psString select = pxDataGet("xcfftool_defineforstacks.sql"); - if (!select) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(xccalWhereMD); - return false; - } - - psString where = NULL; - psString whereClause = psDBGenerateWhereConditionSQL(xccalWhereMD, NULL); - psStringAppend(&where, "\nAND %s", whereClause); - psStringAppend(&select, " %s", where); - psFree(whereClause); - psFree(xccalWhereMD); - - psString joinHook = NULL; - if (!rerun) { - psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = xccalRun.xccal_id"); - psStringAppend(&joinHook, "\n %s\nAND xcForceRun.label = '%s'", where, label); - psStringAppend(&select, "\nAND ff_id IS NULL"); - } - - if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(select); - return false; - } - psFree(select); - psFree(joinHook); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - psFree(where); - return false; - } - if (!psArrayLength(output)) { - psWarning("xcfftool: no rows found"); - psFree(output); - psFree(where); - return true; - } - - if (pretend) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - psFree(where); - return false; - } - psFree(output); - psFree(where); - return true; - } - - psString warpQueryTemplate = pxDataGet("xcfftool_defineforstacks_select_warps.sql"); - - for (long i = 0; i < output->n; i++) { - psMetadata *row = output->data[i]; // Row from select - bool status; - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - return false; - } - - psS64 xccal_id = psMetadataLookupS64(&status, row, "xccal_id"); - - psString path_base = NULL; - if (sources_path_base) { - path_base = sources_path_base; - } else { - path_base = psMetadataLookupStr(&status, row, "path_base"); - psAssert(status, "failed to find xccal path_base?"); - } - - psString xccal_data_group = psMetadataLookupStr(&status, row, "data_group"); - - psString query = NULL; - psStringAppend(&query, warpQueryTemplate, xccal_id); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - // Find the warps for this skycell and filter combination - psArray *warpOutput = p_psDBFetchResult(config->dbh); - if (!warpOutput) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - return false; - } - if (!psArrayLength(warpOutput)) { - // no warps for this xccal. Suprise? - psFree(warpOutput); - psFree(query); - continue; - } - - // create a staticskyRun - if (!xcForceRunInsert(config->dbh, - 0x0, // ff_id - xccal_id, - path_base, - "new", // state - workdir, - label, - data_group ? data_group : (xccal_data_group ? xccal_data_group : label), - dist_group, - note, - reduction, - NULL // registered - ) - ) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - return false; - } - - psS64 ff_id = psDBLastInsertID(config->dbh); - - for (int j = 0; j < warpOutput->n; j++) { - psMetadata *warpRow = warpOutput->data[j]; - psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); - - if (!xcForceInputInsert(config->dbh, - ff_id, - warp_id) - ) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpOutput); - psFree(output); - return false; - } - } - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(warpOutput); - psFree(output); - return false; - } - psFree(warpOutput); - } - psFree(output); - - return true; -} - static bool updaterunMode(pxConfig *config) { @@ -570,5 +360,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "xcForceRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "xcForceRun.data_group", "=="); @@ -585,5 +375,9 @@ } - psString query = psStringCopy("UPDATE xcForceRun JOIN xccalRun USING(xccal_id) JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id)"); + psString query = pxDataGet("xcfftool_updaterun.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } // pxUpdateRun gets parameters from config->args and updates