Index: /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42596) @@ -581,3 +581,13 @@ xcskytool_result.sql \ xcskytool_revert.sql \ - xcskytool_revert_update.sql + xcskytool_revert_update.sql \ + xcfftool_definebyquery_pt1.sql \ + xcfftool_definebyquery_pt2.sql \ + xcfftool_definebyquery_select_warps.sql \ + xcfftool_result.sql \ + xcfftool_revert.sql \ + xcfftool_revertsummary.sql \ + xcfftool_summaryinputs.sql \ + xcfftool_summary.sql \ + xcfftool_toadvance.sql \ + xcfftool_todo.sql Index: anches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery.sql (revision 42595) +++ (revision ) @@ -1,30 +1,0 @@ -SELECT - xccalRun.xccal_id, - xccalResult.path_base, - xccalRun.data_group, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter -FROM %s.xccalRun - JOIN %s.xccalResult USING (xccal_id) - JOIN %s.xcstackInputSkyfile USING (xcstack_id) -- exposes the stack_id - JOIN %s.xcstackCamera USING (camera_id) -- select the camera matching this db - JOIN stackRun USING (stack_id) - JOIN skycell USING (tess_id, skycell_id) - -- join hook %s - -- for no-rerun: - -- LEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id -WHERE - -- the first %s below should be filled with the xcamera database - -- the second should be the name of the camera corresponding to the - -- database in which the tool is run - %s.xcstackCamera.camera = '%s' AND - %s.xccalRun.state = 'full' AND - %s.xccalResult.quality = 0 --- 'where' string is appended here - -- for no-rerun, also append: - -- AND xcff_id IS NULL - --- this query is run in the database in which the original stacks were made (e.g., gpc1) --- the external database (%s in front of the xcstackRun & xccalRun tables) refers to the database --- used to generate the xcstacks (e.g., xcamera) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt1.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt1.sql (revision 42596) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt1.sql (revision 42596) @@ -0,0 +1,19 @@ +SELECT + xccalRun.xccal_id, + xccalResult.path_base, + xccalRun.data_group, + stackRun.tess_id, + stackRun.stack_id, + stackRun.skycell_id, + stackRun.filter +FROM %s.xccalRun + JOIN %s.xccalResult USING (xccal_id) -- needed for input sources path_base + JOIN %s.xcstackInputSkyfile USING (xcstack_id) -- exposes the stack_id + JOIN %s.xcstackCamera USING (camera_id) -- select the camera matching this db + JOIN stackRun USING (stack_id) -- needed for filter + JOIN skycell USING (tess_id, skycell_id) -- needed to limit by coords + +-- this query is joined to part 2 which supplies WHERE conditions, but there may be an optional +-- LEFT JOIN between the two parts for the -no-rerun case + +-- the 4 string formats are replaced with the xcamera database name Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt2.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt2.sql (revision 42596) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_pt2.sql (revision 42596) @@ -0,0 +1,10 @@ +WHERE + -- the first string format below should be filled with the xcamera database + -- the second should be the name of the camera corresponding to the + -- database in which the tool is run + %s.xcstackCamera.camera_id = %lld +AND %s.xccalRun.state = 'full' +AND %s.xccalResult.quality = 0 +-- 'where' string is appended here +-- for no-rerun, this also append: +-- AND xcff_id IS NULL Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_select_warps.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_select_warps.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_definebyquery_select_warps.sql (revision 42596) @@ -1,9 +1,8 @@ SELECT warp_id -FROM warpRun join warpSkyfile using(warp_id, tess_id) - JOIN fakeRun using(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - JOIN skycell ON skycell.tess_id = warpSkyfile.tess_id AND skycell.skycell_id = warpSkyfile.skycell_id -WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' - AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 +FROM stackInputSkyfile +JOIN warpRun using (warp_id) +JOIN warpSkyfile using(warp_id, tess_id) +WHERE (stack_id = %lld) +AND warpRun.state = 'full' +AND warpSkyfile.quality = 0 +AND warpSkyfile.fault = 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_result.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_result.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_result.sql (revision 42596) @@ -1,22 +1,19 @@ SELECT - fullForceResult.*, - fullForceRun.state, - fullForceRun.label, - fullForceRun.data_group, - fullForceRun.xccal_id, - fullForceRun.workdir, - fullForceRun.sources_path_base, + xcForceResult.*, + xcForceRun.state, + xcForceRun.label, + xcForceRun.data_group, + xcForceRun.xccal_id, + xcForceRun.workdir, + xcForceRun.sources_path_base, CONCAT_WS('.', warp_id, skycell_id) AS dist_component, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter, + xcForceRun.tess_id, + xcForceRun.skycell_id, warpImfile.warp_skyfile_id, rawExp.exp_id, rawExp.exp_name, rawExp.camera -FROM fullForceRun - JOIN fullForceResult USING(ff_id) - JOIN xccalRun using(xccal_id) - JOIN stackRun using(stack_id) +FROM xcForceRun + JOIN xcForceResult USING(xcff_id) JOIN warpRun using(warp_id, tess_id) JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revert.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revert.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revert.sql (revision 42596) @@ -1,7 +1,4 @@ -DELETE FROM fullForceResult -USING fullForceResult, fullForceRun, xccalRun, stackRun, skycell -WHERE fullForceRun.ff_id = fullForceResult.ff_id - AND fullForceResult.fault != 0 - AND fullForceRun.xccal_id = xccalRun.xccal_id - AND xccalRun.stack_id = stackRun.stack_id - AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id +DELETE FROM xcForceResult +USING xcForceResult, xcForceRun +WHERE xcForceRun.xcff_id = xcForceResult.xcff_id + AND xcForceResult.fault != 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revertsummary.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revertsummary.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_revertsummary.sql (revision 42596) @@ -1,7 +1,5 @@ -DELETE FROM fullForceSummary -USING fullForceSummary, fullForceRun, xccalRun, stackRun, skycell -WHERE fullForceRun.ff_id = fullForceSummary.ff_id - AND fullForceSummary.fault != 0 - AND fullForceRun.xccal_id = xccalRun.xccal_id - AND xccalRun.stack_id = stackRun.stack_id - AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id +DELETE FROM xcForceSummary +USING xcForceSummary, xcForceRun, skycell +WHERE xcForceRun.xcff_id = xcForceSummary.xcff_id + AND xcForceSummary.fault != 0 + AND xcForceRun.tess_id = skycell.tess_id AND xcForceRun.skycell_id = skycell.skycell_id Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_summary.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_summary.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_summary.sql (revision 42596) @@ -1,25 +1,15 @@ SELECT DISTINCT - fullForceSummary.*, - fullForceRun.state, - fullForceRun.label, - fullForceRun.data_group, - fullForceRun.xccal_id, - fullForceRun.workdir, - fullForceRun.sources_path_base, + xcForceSummary.*, + xcForceRun.state, + xcForceRun.label, + xcForceRun.data_group, + xcForceRun.xccal_id, + xcForceRun.workdir, + xcForceRun.sources_path_base, + xcForceRun.tess_id, + xcForceRun.skycell_id, 'summary' AS dist_component, - stackRun.stack_id, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter, - rawExp.camera -FROM fullForceRun - JOIN fullForceSummary USING(ff_id) - JOIN fullForceInput USING(ff_id) - JOIN xccalRun using(xccal_id) - JOIN stackRun using(stack_id) - JOIN warpRun using(warp_id, tess_id) - JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) - JOIN fakeRun using(fake_id) - JOIN camRun using(cam_id) - JOIN chipRun using(chip_id) - JOIN rawExp using(exp_id) +FROM xcForceRun + JOIN xcForceSummary USING(xcff_id) + JOIN xcForceInput USING(xcff_id) + JOIN skycell USING (tess_id, skycell_id) Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_toadvance.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_toadvance.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_toadvance.sql (revision 42596) @@ -1,22 +1,15 @@ SELECT - fullForceRun.*, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter, - rawExp.camera -FROM fullForceRun - JOIN fullForceInput USING(ff_id) - JOIN xccalRun USING(xccal_id) - JOIN stackRun USING(stack_id) - JOIN warpRun USING(warp_id, tess_id) - JOIN fakeRun USING(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - LEFT JOIN fullForceResult USING(ff_id, warp_id) - LEFT JOIN fullForceSummary USING(ff_id) -WHERE fullForceRun.state = 'new' AND fullForceSummary.ff_id IS NULL + xcForceRun.xcff_id, + xcForceRun.tess_id, + xcForceRun.skycell_id, + xcForceRun.workdir, + xcForceRun.reduction +FROM xcForceRun + JOIN xcForceInput USING(xcff_id) + LEFT JOIN xcForceResult USING(xcff_id, warp_id) + LEFT JOIN xcForceSummary USING(xcff_id) +WHERE xcForceRun.state = 'new' AND xcForceSummary.ff_id IS NULL -- WHERE hook %s - GROUP BY ff_id - HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id) - AND SUM(fullForceResult.fault) = 0 + GROUP BY xcff_id + HAVING COUNT(xcForceInput.warp_id) = COUNT(xcForceResult.warp_id) + AND SUM(xcForceResult.fault) = 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_todo.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_todo.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_todo.sql (revision 42596) @@ -1,22 +1,12 @@ SELECT - fullForceRun.*, - fullForceInput.warp_id, - stackRun.tess_id, - stackRun.skycell_id, - stackRun.filter, - rawExp.camera, + xcForceRun.*, + xcForceInput.warp_id, warpSkyfile.path_base AS warp_path_base -FROM fullForceRun - JOIN fullForceInput USING(ff_id) - JOIN xccalRun USING(xccal_id) - JOIN stackRun USING(stack_id) +FROM xcForceRun + JOIN xcForceInput USING(xcff_id) JOIN warpRun USING(warp_id, tess_id) JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) - JOIN fakeRun USING(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - LEFT JOIN fullForceResult USING(ff_id, warp_id) -WHERE fullForceRun.state = 'new' - AND fullForceResult.ff_id IS NULL + LEFT JOIN xcForceResult USING(xcff_id, warp_id) +WHERE xcForceRun.state = 'new' + AND xcForceResult.xcff_id IS NULL AND warpSkyfile.data_state = 'full' Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcfftool_updaterun.sql (revision 42596) @@ -1,7 +1,2 @@ UPDATE xcForceRun -JOIN %s.xccalRun USING(xccal_id) -JOIN %s.xccalResult USING (xccal_id) -JOIN %s.xcstackInputSkyfile USING (xcstack_id) -- exposes the stack_id -JOIN %s.xcstackCamera USING (camera_id) -- select the camera matching this db -JOIN stackRun USING (stack_id) -JOIN skycell USING(tess_id, skycell_id)"); +JOIN skycell USING(tess_id, skycell_id); Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_definebyquery_select.sql (revision 42596) @@ -22,7 +22,7 @@ LEFT JOIN ( SELECT DISTINCT xcsky_id, - xcstackRun.skycell_id, - xcstackRun.tess_id, - xcstackRun.data_group, + MIN(xcstackRun.skycell_id) as skycell_id, + MIN(xcstackRun.tess_id) as tess_id, + MIN(xcstackRun.data_group) as data_group, COUNT(xcstack_id) AS num_filter FROM xcskyRun Index: /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/share/xcskytool_todo.sql (revision 42596) @@ -1,2 +1,3 @@ + -- we implicitly assume that all xcstackRuns used for a xcskyRun have the same tess_id and skycell_id SELECT xcskyRun.xcsky_id, @@ -5,6 +6,6 @@ xcskyRun.label, xcskyRun.state, - MIN(xcstackRun.tess_id) as tess_id, -- we implicitly assume that all xcstackRuns used for a xcskyRun have the same tess_id - MIN(xcstackRun.skycell_id) as skycell_id, -- and skycell_id + MIN(xcstackRun.tess_id) as tess_id, + MIN(xcstackRun.skycell_id) as skycell_id, TRUNCATE(MIN(skycell.radeg)/15., 4) as rahours, MIN(skycell.radeg) as radeg, Index: /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/src/Makefile.am (revision 42596) @@ -26,4 +26,5 @@ xcstacktool \ xcskytool \ + xcfftool \ warptool \ receivetool \ @@ -86,4 +87,5 @@ xcstacktool.h \ xcskytool.h \ + xcfftool.h \ staticskytool.h \ warptool.h \ @@ -261,4 +263,10 @@ xcskytoolConfig.c +xcfftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +xcfftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +xcfftool_SOURCES = \ + xcfftool.c \ + xcfftoolConfig.c + pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.c (revision 42596) @@ -33,5 +33,4 @@ static bool definebyqueryMode(pxConfig *config); -static bool defineforstacksMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todoMode(pxConfig *config); @@ -48,6 +47,6 @@ static bool importrunMode(pxConfig *config); -static bool setxcForceRunState(pxConfig *config, psS64 sky_id, const char *state); -static psS64 getCameraIDfromName(pxConfig *config, const char *camera); +static bool setxcForceRunState(pxConfig *config, psS64 xcff_id, const char *state); +static psS64 getCameraIDfromName(pxConfig *config, const char *dbname, const char *camera); # define MODECASE(caseName, func) \ @@ -70,5 +69,4 @@ switch (config->mode) { MODECASE(XCFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(XCFFTOOL_MODE_DEFINEFORSTACKS, defineforstacksMode); MODECASE(XCFFTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(XCFFTOOL_MODE_TODO, todoMode); @@ -110,17 +108,33 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + // I need the name of the current working database + psMetadataItem *dbnameMD = pmConfigUserSite(config->modules, "DBNAME", PS_DATA_STRING); + char *dbname = dbnameMD->data.str; + + // required: PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + // we have two relevant databases: the current working database (e.g., gpc1) and the cross-camera + // database used to generate the cross-camera stacks (e.g., xcamera). the cross-camera database + // name is supplied with -xcamera as an argument. the current working database is part of the + // database config info + // The xcamera must be supplied as a string, then the camera_id is selected - // from the table xcstackCamera. + // from the table xcstackCamera. PXOPT_LOOKUP_STR(xcamera, config->args, "-xcamera", true, false); - // get camera_id for the working database: - // XXX need to extract the camera name (-dbname) from the config - psS64 camera_id = getCameraIDfromName(config, config->dbh->camera); + // get camera_id for the working database from the xcamera database + psS64 camera_id = getCameraIDfromName(config, xcamera, dbname); if (!camera_id) { - psError(PXTOOLS_ERR_SYS, false, "failed to find xcamera in database"); - return false; + psError(PXTOOLS_ERR_SYS, false, "failed to find camera %s in database", dbname); + return false; + } + // get xcamera_id for the cross-camera database from the working database + psS64 xcamera_id = getCameraIDfromName(config, dbname, xcamera); + if (!xcamera_id) { + psError(PXTOOLS_ERR_SYS, false, "failed to find camera %s in database", xcamera); + return false; } @@ -131,10 +145,7 @@ PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - // XXX is needed or wanted? - // PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); - // first, we need to select the xccalRun(s) which need to have xcForceRuns. we can limit the selection based on // qualities of the xccalRun or the xcstackRun. There will be one xcForceRun for each - // identified xccalRun. + // identified xccalRun (there may be multiple xcForceRuns with different labels) psMetadata *xccalWhereMD = psMetadataAlloc(); @@ -149,7 +160,8 @@ } - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id", "xcstackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id", "xcstackRun.tess_id", "=="); - pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter", "xcstackRun.filter", "LIKE"); + // the select query does not join to xcstackRun, but it does join to stackRun + PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, xccalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, xccalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, xccalWhereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -167,4 +179,5 @@ pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group", "warpRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); // XXX is is really necessary to limit the warps? if (!psListLength(warpWhereMD->list)) { @@ -174,5 +187,4 @@ return false; } - PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -180,6 +192,8 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - psString select = pxDataGet("xcfftool_definebyquery.sql"); - if (!select) { + psString select = NULL; + + psString query_pt1 = pxDataGet("xcfftool_definebyquery_pt1.sql"); + if (!query_pt1) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(xccalWhereMD); @@ -188,26 +202,36 @@ } + // query_pt1 has 4 places where the xcamera database name must be supplied + psStringAppend (&select, query_pt1, xcamera, xcamera, xcamera, xcamera); + psFree (query_pt1); + + if (!rerun) { + psStringAppend(&select, "LEFT JOIN xcForceRun ON (xcForceRun.xccal_id = %s.xccalRun.xccal_id) AND (xcForceRun.label = '%s')\n", xcamera, label); + } + + psString query_pt2 = pxDataGet("xcfftool_definebyquery_pt2.sql"); + if (!query_pt2) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(xccalWhereMD); + psFree(warpWhereMD); + psFree(select); + return false; + } + + // query_pt2 has 3 places where the xcamera database name must be supplied and 1 for this camera + psStringAppend (&select, query_pt2, xcamera, (long long) camera_id, xcamera, xcamera); + psFree (query_pt2); + psString where = NULL; psString whereClause = psDBGenerateWhereConditionSQL(xccalWhereMD, NULL); - psStringAppend(&where, "\nAND %s", whereClause); - psStringAppend(&select, " %s", where); + psStringAppend(&select, "\n AND %s", whereClause); psFree(whereClause); psFree(xccalWhereMD); - psString joinHook = NULL; - if (!rerun) { - psStringAppend(&joinHook, "\nLEFT JOIN xcForceRun ON xcForceRun.xccal_id = %s.xccalRun.xccal_id", xcamera); - } - - psStringAppend(&joinHook, "\n %s, where); - - if ( - psStringAppend(&joinHook, "\n %s\nAND xcForceRun.label = '%s'", where, label); - psStringAppend(&select, "\nAND ff_id IS NULL"); - } - - // XXX need to add the xcamera name here to match definebyquery.sql - if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { + // accept only the entries without a match in the target label + if (!rerun) psStringAppend(&select, "\n AND xcff_id IS NULL\n"); + + if (!p_psDBRunQuery(config->dbh, select)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(select); @@ -215,5 +239,4 @@ } psFree(select); - psFree(joinHook); psArray *output = p_psDBFetchResult(config->dbh); @@ -251,8 +274,11 @@ } + // this query selects all of the warps where were inputs to the given stackRun, + // with restrictions based on (quality, fault, warpRun.state, and other supplied conditions) + // there is no limit to the version of the warp analyss, to psString warpQueryTemplate = pxDataGet("xcfftool_definebyquery_select_warps.sql"); whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL); - psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause); + psStringAppend(&warpQueryTemplate, "\n AND %s", whereClause); for (long i = 0; i < output->n; i++) { @@ -268,20 +294,12 @@ // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); psS64 xccal_id = psMetadataLookupS64(&status, row, "xccal_id"); - - psString path_base = NULL; - if (sources_path_base) { - path_base = sources_path_base; - } else { - path_base = psMetadataLookupStr(&status, row, "path_base"); - psAssert(status, "failed to find xccal path_base?"); - } - + psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id"); + psString path_base = psMetadataLookupStr(&status, row, "path_base"); psString xccal_data_group = psMetadataLookupStr(&status, row, "data_group"); psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); - psString filter = psMetadataLookupStr(&status, row, "filter"); psString query = NULL; - psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter); + psStringAppend(&query, warpQueryTemplate, (long long) stack_id); if (!p_psDBRunQuery(config->dbh, query)) { @@ -315,15 +333,17 @@ // create an xcForceRun if (!xcForceRunInsert(config->dbh, - 0x0, // ff_id - xccal_id, - path_base, - "new", // state - workdir, - label, - data_group ? data_group : (xccal_data_group ? xccal_data_group : label), - dist_group, - note, - reduction, - NULL // registered + 0x0, // xcff_id + xccal_id, + path_base, + "new", // state + workdir, + label, + skycell_id, + tess_id, + data_group ? data_group : (xccal_data_group ? xccal_data_group : label), + dist_group, + note, + reduction, + NULL // registered ) ) { @@ -342,6 +362,5 @@ psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); - if (!xcForceInputInsert(config->dbh, ff_id, warp_id, xcamera_id) - ) { + if (!xcForceInputInsert(config->dbh, xcff_id, warp_id, xcamera_id)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); @@ -372,9 +391,11 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "xcForceRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "xcForceRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "xcForceRun.state", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); if (!pxskycellAddWhere(config, where)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -387,12 +408,8 @@ } - psString query = pxDataGet("xcfftool_updaterun.sql"); - if (!select) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } + psString query = psStringCopy ("UPDATE xcForceRun JOIN skycell USING(tess_id, skycell_id)"); // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "xcForceRun", "ff_id", "xcForceResult", true, false); + bool result = pxUpdateRun(config, where, &query, "xcForceRun", "xcff_id", "xcForceResult", true, false); psFree(query); psFree(where); @@ -407,6 +424,6 @@ psMetadata *whereMD = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); - pxAddLabelSearchArgs (config, whereMD, "-label", "xcForceRun.label", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-xcff_id", "xcff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "xcForceRun.label", "=="); pxskycellAddWhere(config, whereMD); @@ -482,5 +499,5 @@ PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(xcff_id, config->args, "-xcff_id", true, false); PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); @@ -497,5 +514,5 @@ // XXX not sure we need a transaction here... if (!xcForceResultInsert(config->dbh, - ff_id, + xcff_id, warp_id, path_base, @@ -521,5 +538,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.xcff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "xcForceResult.warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); @@ -608,5 +625,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.ff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "xcForceResult.warp_id", "=="); pxAddLabelSearchArgs(config, where, "-label", "xcForceRun.label", "=="); @@ -660,5 +677,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id", "=="); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); @@ -672,20 +689,9 @@ } -static bool setxcForceRunState(pxConfig *config, psS64 ff_id, const char *state) -{ - psString query = "UPDATE xcForceRun SET state = 'full' WHERE ff_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, ff_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - static bool toadvanceMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *whereMD = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-xcff_id", "xcff_id", "=="); pxAddLabelSearchArgs (config, whereMD, "-label", "xcForceRun.label", "=="); pxskycellAddWhere(config, whereMD); @@ -765,5 +771,5 @@ PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(xcff_id, config->args, "-xcff_id", true, false); // optional @@ -783,5 +789,5 @@ if (!xcForceSummaryInsert(config->dbh, - ff_id, + xcff_id, path_base, dtime_script, @@ -799,5 +805,5 @@ if (!fault) { - if (!setxcForceRunState(config, ff_id, "full")) { + if (!setxcForceRunState(config, xcff_id, "full")) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -817,83 +823,14 @@ } -static bool revertsummaryMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "=="); - pxAddLabelSearchArgs(config, where, "-label", "xcForceRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "xcForceSummary.fault", "=="); - - if (!pxskycellAddWhere(config, where)) { - psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); - return false; - } - - if (!psListLength(where->list)) { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } - - // Delete product - psString delete = pxDataGet("xcfftool_revertsummary.sql"); - if (!delete) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&delete, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, delete)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(delete); - psFree(where); - return false; - } - psFree(delete); - - int numRows = psDBAffectedRows(config->dbh); // Number of row affected - psLogMsg("xcfftool", PS_LOG_INFO, "Deleted %d rows", numRows); - - psFree(where); - return true; -} -static bool updatesummaryMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); - - - if (!pxSetFaultCode(config->dbh, "xcForceSummary", where, fault, quality)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree (where); - return false; - } - psFree (where); - return true; -} static bool summaryMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-ff_id", "xcForceRun.ff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-warp_id", "xcForceInput.warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_S64(config->args, where, "-xccal_id", "xcForceRun.xccal_id", "=="); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.xcff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xccal_id", "xcForceRun.xccal_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "xcForceRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "xcForceRun.data_group", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "xcForceRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "xcForceRun.skycell_id", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "xcForceSummary.fault", "=="); pxskycellAddWhere(config, where); @@ -966,17 +903,80 @@ } +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.xcff_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "xcForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "xcForceSummary.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("xcfftool_revertsummary.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("xcfftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} + +static bool updatesummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcff_id", "=="); + + if (!pxSetFaultCode(config->dbh, "xcForceSummary", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} + bool exportrunMode(pxConfig *config) { - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; + typedef struct ExportTable { + char tableName[80]; + char tableSQL[80]; + } ExportTable; PS_ASSERT_PTR_NON_NULL(config, NULL); - // PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true, false); PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); -// PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); FILE *f = fopen (outfile, "w"); @@ -994,8 +994,8 @@ ExportTable tables [] = { - {"xcForceRun", "xcfftool_export_run.sql"}, - {"xcForceInput", "xcfftool_export_input.sql"}, - {"xcForceResult", "xcfftool_export_result.sql"}, - {"xcForceSummary", "xcfftool_export_summary.sql"}, + {"xcForceRun", "SELECT xcForceRun.* FROM xcForceRun"}, + {"xcForceInput", "SELECT xcForceInput.* FROM xcForceInput"}, + {"xcForceResult", "SELECT xcForceResult.* FROM xcForceResult"}, + {"xcForceSummary", "SELECT xcForceSummary.* FROM xcForceSummary"}, }; @@ -1003,48 +1003,44 @@ for (int i=0; i < numTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psError(PS_ERR_UNKNOWN, true, "no rows found"); - psFree(output); - return false; - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); + psString query = psStringCopy(tables[i].tableSQL); + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); } @@ -1054,4 +1050,5 @@ } +// XXX this is coded in a very strange way bool importrunMode(pxConfig *config) { @@ -1067,9 +1064,4 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - -#ifdef notdef - fprintf (stderr, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); -#endif if (!pxCheckImportVersion(config, input)) { @@ -1143,11 +1135,11 @@ // We expect the camera to be defined, so an invalid name or a missing camera // results in a 0 return value which should be treated as an error. -static psS64 getCameraIDfromName(pxConfig *config, const char *camera) +static psS64 getCameraIDfromName(pxConfig *config, const char *dbname, const char *camera) { PS_ASSERT_PTR_NON_NULL(camera, false); - char *query = "SELECT camera_id FROM xcstackCamera WHERE camera = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, camera)) { - psError(PS_ERR_UNKNOWN, false, "failed to select camera_id for camera %s", camera); + char *query = "SELECT camera_id FROM %s.xcstackCamera WHERE camera = '%s'"; + if (!p_psDBRunQueryF(config->dbh, query, dbname, camera)) { + psError(PS_ERR_UNKNOWN, false, "failed to select camera_id for camera %s from database %s", camera, dbname); return 0; } @@ -1197,3 +1189,13 @@ } - +static bool setxcForceRunState(pxConfig *config, psS64 xcff_id, const char *state) +{ + psString query = "UPDATE xcForceRun SET state = 'full' WHERE xcff_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, xcff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.h =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.h (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcfftool.h (revision 42596) @@ -26,5 +26,4 @@ XCFFTOOL_MODE_NONE = 0x0, XCFFTOOL_MODE_DEFINEBYQUERY, - XCFFTOOL_MODE_DEFINEFORSTACKS, XCFFTOOL_MODE_UPDATERUN, XCFFTOOL_MODE_TODO, Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcfftoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcfftoolConfig.c (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcfftoolConfig.c (revision 42596) @@ -48,19 +48,20 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "search by warp label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "search by warp data_group (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "search by warp ID", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); - - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-xcamera", 0, "specify the cross-camera database", false); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_xccal_label", PS_META_DUPLICATE_OK, "search by xccal label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_xccal_data_group", PS_META_DUPLICATE_OK, "search by xccal data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_xccal_id", 0, "search by xccal ID", 0); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "limits warps by label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "limit warps by data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "limit warps by warp ID", 0); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "limit by skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "limit by tess_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "limit by stack filter (LIKE comparison, multiple OK)", NULL); pxskycellAddArguments(definebyqueryArgs); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); @@ -72,34 +73,13 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - // -defineforstacks - psMetadata *defineforstacksArgs = psMetadataAlloc(); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); - psMetadataAddS64(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); - - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); - psMetadataAddF32(defineforstacksArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); - - pxskycellAddArguments(defineforstacksArgs); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); - psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); - psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); - psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); - psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess_id", NULL); pxskycellAddArguments(updaterunArgs); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); @@ -111,5 +91,5 @@ // -todo psMetadata *todoArgs = psMetadataAlloc(); - psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); pxskycellAddArguments(todoArgs); @@ -119,5 +99,5 @@ // -addresult psMetadata *addresultArgs = psMetadataAlloc(); - psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0); psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); @@ -130,11 +110,10 @@ // -result psMetadata *resultArgs= psMetadataAlloc(); - psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcforce ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by xccal ID", 0); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", 0); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); - psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); @@ -149,5 +128,5 @@ // -revert psMetadata *revertArgs= psMetadataAlloc(); - psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); @@ -157,5 +136,5 @@ // -updateresult psMetadata *updateresultArgs = psMetadataAlloc(); - psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0); psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); @@ -164,5 +143,5 @@ // -toadvance psMetadata *toadvanceArgs = psMetadataAlloc(); - psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); pxskycellAddArguments(toadvanceArgs); @@ -172,5 +151,5 @@ // -addsummary psMetadata *addsummaryArgs = psMetadataAlloc(); - psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0); psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); @@ -180,7 +159,27 @@ psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + // -summary + psMetadata *summaryArgs= psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-xccal_id", 0, "search by skycal ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + pxskycellAddArguments(summaryArgs); + psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsummary + psMetadata *revertsummaryArgs= psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "search by xcff ID", 0); + psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertsummaryArgs); + // -updatesummary psMetadata *updatesummaryArgs = psMetadataAlloc(); - psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-xcff_id", 0, "define xcff ID (required)", 0); psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); @@ -190,32 +189,7 @@ psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); - // -summary - psMetadata *summaryArgs= psMetadataAlloc(); - psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); - psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); - psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", 0); - psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); - psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); - psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); - pxskycellAddArguments(summaryArgs); - psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -revertsummary - psMetadata *revertsummaryArgs= psMetadataAlloc(); - psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); - psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); - psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); - pxskycellAddArguments(revertsummaryArgs); - // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); - psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0, "export this full force ID (required)", 0); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-xcff_id", 0, "export this xcff ID (required)", 0); psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -226,6 +200,4 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - - psFree(now); @@ -233,6 +205,5 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "Define a new full force run", XCFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-defineforstacks", "Define a new full force runs for inputs to stacks", XCFFTOOL_MODE_DEFINEFORSTACKS, defineforstacksArgs); + PXOPT_ADD_MODE("-definebyquery", "Define a new xcff run", XCFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "Update a run", XCFFTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-todo", "Get runs to do", XCFFTOOL_MODE_TODO, todoArgs); Index: /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42595) +++ /branches/eam_branches/ipp-20230313/ippTools/src/xcskytool.c (revision 42596) @@ -137,5 +137,29 @@ psAssert (filters->type == PS_DATA_METADATA_MULTI, "-select_filter should be a multi container"); psAssert (filters->data.list->n, "-select_filter should at least have a place-holder"); + +# if (0) + // XXX 'filters' is a psMetadataItem of type MULTI. Here is code to iterate over the list of items: + psListIterator *myIter = psListIteratorAlloc(filters->data.list, PS_LIST_HEAD, false); + psMetadataItem *myItem = NULL; + while ((myItem = psListGetAndIncrement (myIter))) { + fprintf (stderr, "type: %d\n", (int) myItem->type); + } +# endif + int num_filter = filters->data.list->n; + + // if -select_filter was not supplied, then the 'filters' container will have a single entry + // with a value of NULL + if (num_filter == 1) { + // psListIterator *myIter = psListIteratorAlloc(filters->data.list, PS_LIST_HEAD, false); + // psMetadataItem *myItem = psListGetAndIncrement (myIter); + + psMetadataItem *myItem = psListGet (filters->data.list, 0); + psAssert (myItem->type == PS_DATA_STRING, "unexpected data type for -select_filter"); + if (myItem->data.str == NULL) { + psError(PXTOOLS_ERR_CONFIG, false, "at least one -select_filter must be supplied"); + return false; + } + } PXOPT_LOOKUP_BOOL(group_by_data_group, config->args, "-group_by_data_group", false);