Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/Makefile.am (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/Makefile.am (revision 36680) @@ -33,5 +33,7 @@ vptool \ sctool \ - releasetool + releasetool \ + fftool \ + remotetool pkginclude_HEADERS = \ @@ -87,5 +89,7 @@ vptool.h \ sctool.h \ - releasetool.h + releasetool.h \ + fftool.h \ + remotetool.h lib_LTLIBRARIES = libpxtools.la @@ -332,4 +336,16 @@ releasetoolConfig.c +fftool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +fftool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +fftool_SOURCES = \ + fftool.c \ + fftoolConfig.c + +remotetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +remotetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +remotetool_SOURCES = \ + remotetool.c \ + remotetoolConfig.c + clean-local: -rm -f TAGS Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/chiptool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/chiptool.c (revision 36680) @@ -1255,5 +1255,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); psString query = pxDataGet("chiptool_pendingcleanuprun.sql"); @@ -1269,4 +1269,6 @@ } psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, chip_id"); // treat limit == 0 as "no limit" Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.c (revision 36680) @@ -29,4 +29,6 @@ static bool searchMode(pxConfig *config); static bool selectMode(pxConfig *config); +static bool showMode(pxConfig *config); + # define MODECASE(caseName, func) \ @@ -50,4 +52,5 @@ MODECASE(DETSELECT_MODE_SEARCH, searchMode); MODECASE(DETSELECT_MODE_SELECT, selectMode); + MODECASE(DETSELECT_MODE_SHOW, showMode); default: psAbort("invalid option (this should not happen)"); @@ -241,2 +244,64 @@ return true; } + +static bool showMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-type", "det_type", "=="); + + psString query = pxDataGet("detselect_show.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + psStringAppend(&query, " ORDER BY registered DESC, iteration DESC"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return(true); +} Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.h (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselect.h (revision 36680) @@ -27,4 +27,5 @@ DETSELECT_MODE_SEARCH, DETSELECT_MODE_SELECT, + DETSELECT_MODE_SHOW } detselectMode; Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselectConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselectConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/detselectConfig.c (revision 36680) @@ -65,4 +65,12 @@ psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -show + psMetadata *showArgs = psMetadataAlloc(); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-det_type", 0, "search by detrend type", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-type", 0, "search by detrend type (alias for -det_type)", NULL); + psMetadataAddBool(showArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadata *argSets = psMetadataAlloc(); @@ -71,4 +79,5 @@ PXOPT_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); PXOPT_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + PXOPT_ADD_MODE("-show", "show all active detrends of this type with constraints", DETSELECT_MODE_SHOW, showArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/difftool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/difftool.c (revision 36680) @@ -1253,4 +1253,5 @@ PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_STR(warp_data_group, config->args, "-data_group", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -1260,4 +1261,10 @@ + if (!data_group) { + if (warp_data_group) { + data_group = warp_data_group; + } + } + // Get query file psString query = pxDataGet("difftool_definewarpstack.sql"); @@ -1342,4 +1349,12 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psString warpDataGroup = psMetadataLookupStr(&mdok, row, "warpDataGroup"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warpDataGroup not found"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1384,5 +1399,5 @@ workdir, label, - data_group ? data_group : label, + data_group ? data_group : warpDataGroup, dist_group, reduction, Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttool.c (revision 36680) @@ -375,7 +375,23 @@ } if (dist_group) { - psStringAppend(&query, " AND (sticskyRun.dist_group = '%s')", dist_group); + psStringAppend(&query, " AND (staticskyRun.dist_group = '%s')", dist_group); } // (static)sky stage doesn't require magic + magic = false; + } else if (!strcmp(stage, "skycal")) { + query = pxDataGet("disttool_definebyquery_skycal.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (label) { + psStringAppend(&query, " AND (skycalRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (skycalRun.dist_group = '%s')", dist_group); + } + // skycal stage doesn't require magic magic = false; } else if (!strcmp(stage, "SSdiff")) { @@ -394,4 +410,21 @@ if (dist_group) { psStringAppend(&query, " AND (diffRun.dist_group = '%s') ", dist_group); + } + + magic = false; + } else if (!strcmp(stage, "ff")) { + runJoinStr = "fullForceRun.ff_id"; + query = pxDataGet("disttool_definebyquery_ff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (fullForceRun.label = '%s') ", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fullForceRun.dist_group = '%s') ", dist_group); } @@ -661,5 +694,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "distRun.data_group", "LIKE"); // require data_group or dist_id to be supplied @@ -743,5 +776,5 @@ query = "DELETE FROM distComponent where dist_id = %" PRId64; - char *query2 = "DELETE from rcDSFileset using distRun, rcDSFileset WHERE distRun.dist_id = rcDSFileset.dist_id AND rcDSFileset.state ='cleaned' AND dist_id = %" PRId64; + char *query2 = "DELETE from rcDSFileset using distRun, rcDSFileset WHERE distRun.dist_id = rcDSFileset.dist_id AND rcDSFileset.state ='cleaned' AND rcDSFileset.dist_id = %" PRId64; for (long i=0; i < psArrayLength(output); i++) { Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttoolConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/disttoolConfig.c (revision 36680) @@ -91,5 +91,5 @@ // -startover psMetadata *startoverArgs = psMetadataAlloc(); - psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group", NULL); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (LIKE comparison)", NULL); psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.c (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.c (revision 36680) @@ -0,0 +1,1103 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "fftool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool todoMode(pxConfig *config); +static bool addresultMode(pxConfig *config); +static bool resultMode(pxConfig *config); +static bool revertMode(pxConfig *config); +static bool updateresultMode(pxConfig *config); +static bool toadvanceMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); +static bool updatesummaryMode(pxConfig *config); +static bool summaryMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); +static bool summaryMode(pxConfig *config); + +static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = fftoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(FFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FFTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(FFTOOL_MODE_TODO, todoMode); + MODECASE(FFTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(FFTOOL_MODE_RESULT, resultMode); + MODECASE(FFTOOL_MODE_REVERT, revertMode); + MODECASE(FFTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(FFTOOL_MODE_TOADVANCE, toadvanceMode); + MODECASE(FFTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(FFTOOL_MODE_REVERTSUMMARY, revertsummaryMode); + MODECASE(FFTOOL_MODE_UPDATESUMMARY, updatesummaryMode); + MODECASE(FFTOOL_MODE_SUMMARY, summaryMode); + MODECASE(FFTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(FFTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + // optional + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + + PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); + + psMetadata *skycalWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); + + if (!psListLength(skycalWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, skycalWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + return false; + } + + psMetadata *warpWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); + psFree(warpWhereMD); + psFree(skycalWhereMD); + return false; + } + PXOPT_COPY_F32(config->args, warpWhereMD, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "rawExp.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString select = pxDataGet("fftool_definebyquery.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psStringAppend(&select, " %s", where); + psFree(whereClause); + psFree(skycalWhereMD); + + psString joinHook = NULL; + if (!rerun) { + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = skycalRun.skycal_id"); + psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); + psStringAppend(&select, "\nAND ff_id IS NULL"); + } + + if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + return false; + } + psFree(select); + psFree(joinHook); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("fftool: no rows found"); + psFree(output); + psFree(where); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString warpQueryTemplate = pxDataGet("fftool_definebyquery_select_warps.sql"); + + whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL); + psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause); + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id"); + + psString path_base = NULL; + if (sources_path_base) { + path_base = sources_path_base; + } else { + path_base = psMetadataLookupStr(&status, row, "path_base"); + psAssert(status, "failed to find skycal path_base?"); + } + + psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + psString filter = psMetadataLookupStr(&status, row, "filter"); + + psString query = NULL; + psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // Find the warps for this skycell and filter combination + psArray *warpOutput = p_psDBFetchResult(config->dbh); + if (!warpOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(warpOutput)) { + // no warps for this skycal. Suprise? + psFree(warpOutput); + psFree(query); + continue; + } + + // create a staticskyRun + if (!fullForceRunInsert(config->dbh, + 0x0, // ff_id + skycal_id, + path_base, + "new", // state + workdir, + label, + data_group ? data_group : (skycal_data_group ? skycal_data_group : label), + dist_group, + note, + reduction, + NULL // registered + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psS64 ff_id = psDBLastInsertID(config->dbh); + + for (int j = 0; j < warpOutput->n; j++) { + psMetadata *warpRow = warpOutput->data[j]; + psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); + + if (!fullForceInputInsert(config->dbh, + ff_id, + warp_id) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + psFree(warpOutput); + } + psFree(output); + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fullForceRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE fullForceRun JOIN skycalRun USING(skycal_id) JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fullForceRun", "ff_id", "fullForceResult", true, false); + psFree(query); + psFree(where); + + return result; +} + + +static bool todoMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_todo.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + // XXX not sure we need a transaction here... + if (!fullForceResultInsert(config->dbh, + ff_id, + warp_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool resultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-quality", "fullForceResult.quality", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_result.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revert.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} + +static bool updateresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + + if (!pxSetFaultCode(config->dbh, "fullForceResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} + +static bool setfullForceRunState(pxConfig *config, psS64 ff_id, const char *state) +{ + psString query = "UPDATE fullForceRun SET state = 'full' WHERE ff_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, ff_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool toadvanceMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_toadvance.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = NULL; + psString temp = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&whereClause, "\n AND %s", temp); + psFree(temp); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // the where clause is required and is added to the query by the "WHERE hook" format string + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fullForceSummaryInsert(config->dbh, + ff_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fault) { + if (!setfullForceRunState(config, ff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change staticskyRun state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceSummary.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revertsummary.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} +static bool updatesummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + + if (!pxSetFaultCode(config->dbh, "fullForceSummary", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool summaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceInput.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceSummary", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +// PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + ExportTable tables [] = { + {"fullForceRun", "fftool_export_run.sql"}, + {"fullForceInput", "fftool_export_input.sql"}, + {"fullForceResult", "fftool_export_result.sql"}, + {"fullForceSummary", "fftool_export_summary.sql"}, + }; + + int numTables = sizeof(tables)/sizeof(tables[0]); + + for (int i=0; i < numTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + int numImportTables = 3; + + char tables[3] [80] = {"fullForceInput", "fullForceResult", "fullForceSummary"}; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + +#ifdef notdef + fprintf (stderr, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + + psMetadataItem *item = psMetadataLookup (input, "fullForceRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceRunRow *fullForceRun = fullForceRunObjectFromMetadata (entry->data.md); + fullForceRunInsertObject (config->dbh, fullForceRun); + + // fprintf (stdout, "---- warp run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + for (int i = 0; i < numImportTables; i++) { + item = psMetadataLookup (input, tables[i]); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + switch (i) { + case 0: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceInputRow *fullForceInput = fullForceInputObjectFromMetadata (entry->data.md); + fullForceInputInsertObject (config->dbh, fullForceInput); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 1: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceResultRow *fullForceResult = fullForceResultObjectFromMetadata (entry->data.md); + fullForceResultInsertObject (config->dbh, fullForceResult); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 2: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceSummaryRow *fullForceSummary = fullForceSummaryObjectFromMetadata (entry->data.md); + fullForceSummaryInsertObject (config->dbh, fullForceSummary); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + } + } + return true; +} Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.h (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftool.h (revision 36680) @@ -0,0 +1,45 @@ +/* + * fftool.h + * + * Copyright (C) 2013 IfA Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef FFTOOL_H +#define FFTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + FFTOOL_MODE_NONE = 0x0, + FFTOOL_MODE_DEFINEBYQUERY, + FFTOOL_MODE_UPDATERUN, + FFTOOL_MODE_TODO, + FFTOOL_MODE_ADDRESULT, + FFTOOL_MODE_RESULT, + FFTOOL_MODE_REVERT, + FFTOOL_MODE_UPDATERESULT, + FFTOOL_MODE_TOADVANCE, + FFTOOL_MODE_ADDSUMMARY, + FFTOOL_MODE_REVERTSUMMARY, + FFTOOL_MODE_UPDATESUMMARY, + FFTOOL_MODE_SUMMARY, + FFTOOL_MODE_EXPORTRUN, + FFTOOL_MODE_IMPORTRUN, +} fftoolMode; + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv); + +#endif // FFTOOL_H Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftoolConfig.c (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/fftoolConfig.c (revision 36680) @@ -0,0 +1,245 @@ +/* + * fftool.c + * + * Copyright (C) 2013 Institute for Astronomy, University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "fftool.h" + +pxConfig *fftoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + (void) now; + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_label", PS_META_DUPLICATE_OK, "search by warp label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_data_group", PS_META_DUPLICATE_OK, "search by warp data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); + + pxskycellAddArguments(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + pxskycellAddArguments(updaterunArgs); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + // -todo + psMetadata *todoArgs = psMetadataAlloc(); + psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(todoArgs); + psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addresult + psMetadata *addresultArgs = psMetadataAlloc(); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -result + psMetadata *resultArgs= psMetadataAlloc(); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-quality", 0, "search by quality value", 0); + pxskycellAddArguments(resultArgs); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revert + psMetadata *revertArgs= psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertArgs); + + // -updateresult + psMetadata *updateresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality code", 0); + + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(toadvanceArgs); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -updatesummary + psMetadata *updatesummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -summary + psMetadata *summaryArgs= psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + pxskycellAddArguments(summaryArgs); + psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsummary + psMetadata *revertsummaryArgs= psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertsummaryArgs); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0, "export this full force ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); +// psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + psFree(now); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "Define a new full force run", FFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "Update a run", FFTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-todo", "Get runs to do", FFTOOL_MODE_TODO, todoArgs); + PXOPT_ADD_MODE("-addresult", "Add result for fullforce run on a warp", FFTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-result", "Get result fullforce run on a warp", FFTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed fullforce run on a warp", FFTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update result for fullforce run on a warp", FFTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-toadvance", "list completed runs to summarize", FFTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-addsummary", "insert summary result", FFTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-updatesummary", "update summary result", FFTOOL_MODE_UPDATESUMMARY, updatesummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-summary", "list summary results", FFTOOL_MODE_SUMMARY, summaryArgs); + PXOPT_ADD_MODE("-exportrun", "list summary results", FFTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "list summary results", FFTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/laptool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/laptool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/laptool.c (revision 36680) @@ -1170,5 +1170,7 @@ // we only want projection cells which do not already of an entry psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n"); - psStringAppend(&query, "\nORDER by projection_cell\n"); + // Projection cell's don't have a convienient order. Queue runs in the order of lap_id + // psStringAppend(&query, "\nORDER by projection_cell\n"); + psStringAppend(&query, "\nORDER by lap_id_0\n"); if (limit) { @@ -1289,4 +1291,5 @@ psFree(whereClause); } + psStringAppend(&query, "\nGROUP by projection_cell ORDER by MIN(lap_id)"); if (limit) { psString limitString = psDBGenerateLimitSQL(limit); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/pstamptool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/pstamptool.c (revision 36680) @@ -1428,5 +1428,5 @@ psFree(where); - psStringAppend(&query, " ORDER BY priority DESC, req_id, dep_id"); + psStringAppend(&query, " GROUP BY dep_id ORDER BY priority DESC, MIN(req_id), dep_id"); // treat limit == 0 as "no limit" Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtool.c (revision 36680) @@ -310,4 +310,9 @@ } + +// CZW: 2013-12-11 A bad exposure caused this code to fail to return any rows. +// I don't clearly see any logic flaws, and so I fixed the issue by +// retricting dateobs_begin to exclude this exposure. Problem for +// another day. static bool pendingburntoolimfileMode(pxConfig *config) { Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtoolConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/regtoolConfig.c (revision 36680) @@ -292,5 +292,5 @@ psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); - psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.c (revision 36680) @@ -47,4 +47,5 @@ static bool updaterelstackMode(pxConfig *config); static bool listrelstackMode(pxConfig *config); +static bool summaryMode(pxConfig *config); static bool definerelgroupMode(pxConfig *config); static bool pendingrelgroupMode(pxConfig *config); @@ -84,4 +85,5 @@ MODECASE(RELEASETOOL_MODE_UPDATERELSTACK, updaterelstackMode); MODECASE(RELEASETOOL_MODE_LISTRELSTACK, listrelstackMode); + MODECASE(RELEASETOOL_MODE_SUMMARY, summaryMode); MODECASE(RELEASETOOL_MODE_DEFINERELGROUP, definerelgroupMode); @@ -180,9 +182,11 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S32(surveyID, config->args, "-set_surveyID", true, false); - PXOPT_LOOKUP_STR(release_name, config->args, "-set_release_name", true, false); + PXOPT_LOOKUP_S32(surveyID, config->args, "-set_surveyID", true, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-set_release_name", true, false); PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", true, false); - PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); - PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); + PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); + PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(ubercal_file, config->args, "-set_ubercal_file", false, false); if (!ippReleaseInsert(config->dbh, @@ -192,5 +196,7 @@ release_state, dataRelease, - priority + priority, + dvodb, + ubercal_file )) { psError(PS_ERR_UNKNOWN, false, "failed to insert ippRelease"); @@ -215,6 +221,8 @@ PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); - if (!release_state && priority == 0 && dataRelease == -1) { - psError(PXTOOLS_ERR_CONFIG, true, "at least one of -set_release_state -set_priority and -set_dataRelease is required\n"); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(ubercal_file, config->args, "-set_ubercal_file", false, false); + if (!release_state && priority == 0 && dataRelease == -1 && !dvodb && !ubercal_file) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -set_release_state -set_priority -set_dvodb -set_ubercal_file and -set_dataRelease is required\n"); return false; } @@ -238,4 +246,12 @@ if (dataRelease != -1) { psStringAppend(&query, "%s dataRelease = %d", sep, dataRelease); + sep = comma; + } + if (dvodb) { + psStringAppend(&query, "%s dvodb = '%s'", sep, dvodb); + sep = comma; + } + if (ubercal_file) { + psStringAppend(&query, "%s ubercal_file = '%s'", sep, ubercal_file); sep = comma; } @@ -366,4 +382,6 @@ PXOPT_LOOKUP_F32(user_zpt_obs, config->args, "-set_zpt_obs", false, false); PXOPT_LOOKUP_F32(user_zpt_stdev, config->args, "-set_zpt_stdev", false, false); + PXOPT_LOOKUP_F32(mcal, config->args, "-set_mcal", false, false); + PXOPT_LOOKUP_S32(ubercal_dist, config->args, "-set_ubercal_dist", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); @@ -456,4 +474,6 @@ zpt_obs, zpt_stdev, + mcal, + ubercal_dist, path_base, fault, @@ -496,10 +516,13 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_U32(flags, config->args, "-set_flags", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); + PXOPT_LOOKUP_STR(path_base,config->args, "-set_path_base", false, false); PXOPT_LOOKUP_F32(zpt_obs, config->args, "-set_zpt_obs", false, false); PXOPT_LOOKUP_F32(zpt_stdev,config->args, "-set_zpt_stdev", false, false); + PXOPT_LOOKUP_F32(mcal, config->args, "-set_mcal", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_S32(ubercal_dist, config->args, "-set_ubercal_dist", false, false); PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); - if (!state && !flags && !path_base && !isfinite(zpt_obs) && !isfinite(zpt_stdev) && !fault && !clearfault) { + if (!state && !flags && !path_base && !isfinite(zpt_obs) && !isfinite(zpt_stdev) &&!isfinite(mcal) + && !fault && !clearfault && !ubercal_dist) { psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n"); return false; @@ -557,4 +580,12 @@ sep = comma; } + if (isfinite(mcal)) { + psStringAppend(&query, "%s relExp.mcal = %f", sep, mcal); + sep = comma; + } + if (ubercal_dist) { + psStringAppend(&query, "%s relExp.ubercal_dist = %d", sep, ubercal_dist); + sep = comma; + } psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -569,5 +600,5 @@ psU64 affected = psDBAffectedRows(config->dbh); - psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " relExp rows", affected); @@ -1049,5 +1080,98 @@ if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC"); + psStringAppend(&query, "\nAND priority > 0 order by priority DESC, stack_id"); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "relStack", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool summaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociation.sass_id", "=="); + PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "relStack.stack_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-skycal_id", "relStack.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-relstack_id", "relStack.relstack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); + pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); +// PXOPT_COPY_STR(config->args, where, "-state", "relStack.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "relStack.filter", "LIKE"); +// PXOPT_COPY_F32(config->args, where, "-mjd_min", "stackSumSkyfile.mjd_obs", ">="); +// PXOPT_COPY_F32(config->args, where, "-mjd_max", "stackSumSkyfile.mjd_obs", "<="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "relStack.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "relStack.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "stackAssociation.projection_cell", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stack_data_group", "stackRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackAssociation.data_group", "LIKE"); +// PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + pxskycellAddWhere(config, where); + +// PXOPT_COPY_F32(config->args, where, "-fwhm_min", "IFNULL(skycalResult.fwhm_major, 999)", ">="); +// PXOPT_COPY_F32(config->args, where, "-fwhm_max", "IFNULL(skycalResult.fwhm_major, 0)", "<="); + + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "=="); + + psString query = pxDataGet("releasetool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (priority_order) { + psStringAppend(&query, "\nAND priority > 0 order by priority DESC, stack_id"); } Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.h (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetool.h (revision 36680) @@ -38,4 +38,5 @@ RELEASETOOL_MODE_TOCALIBSTACK, RELEASETOOL_MODE_LISTRELSTACK, + RELEASETOOL_MODE_SUMMARY, RELEASETOOL_MODE_DEFINERELGROUP, RELEASETOOL_MODE_PENDINGRELGROUP, Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetoolConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/releasetoolConfig.c (revision 36680) @@ -64,4 +64,6 @@ psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define release state", NULL); psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define release priority", 0); + psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvo db name", NULL); + psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_ubercal_file", 0, "define ubercal file name", NULL); // -uptdaterelease @@ -75,4 +77,6 @@ psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define new release priority", 0); psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define data release", -1); + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define new dvo db name", NULL); + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_ubercal_file", 0, "define new ubercal file name", NULL); @@ -108,4 +112,6 @@ psMetadataAddF32(definerelexpArgs, PS_LIST_TAIL, "-set_zpt_obs", 0, "define zero point", NAN); psMetadataAddF32(definerelexpArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define zero point stdev", NAN); + psMetadataAddF32(definerelexpArgs, PS_LIST_TAIL, "-set_mcal", 0, "define mcal", NAN); + psMetadataAddS32(definerelexpArgs, PS_LIST_TAIL, "-set_ubercal_dist", 0, "define ubercal dist", 0); psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state", NULL); psMetadataAddS16(definerelexpArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); @@ -126,4 +132,6 @@ psMetadataAddF32(updaterelexpArgs, PS_LIST_TAIL, "-set_zpt_obs", 0, "define new zero point", NAN); psMetadataAddF32(updaterelexpArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define new zero point stdev", NAN); + psMetadataAddF32(updaterelexpArgs, PS_LIST_TAIL, "-set_mcal", 0, "define mcal", NAN); + psMetadataAddS32(updaterelexpArgs, PS_LIST_TAIL, "-set_ubercal_dist", 0, "define new ubercal dist", 0); psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-set_path_base", 0, "define path_base", NULL); psMetadataAddS16(updaterelexpArgs, PS_LIST_TAIL, "-set_fault", 0, "define new fault", 0); @@ -240,4 +248,33 @@ psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -summary + psMetadata *summaryArgs = psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-sass_id", 0, "select by released SASS ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-relstack_id", 0, "select by released exposure ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "select by stack ID", 0); +// psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "select by skycal ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL); +// psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-state", 0, "select by released stack state", NULL); + + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "select by filter name (LIKE comparison)", NULL); + + pxskycellAddArguments(summaryArgs); + + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "select by projection_cell (LIKE comparision)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL); + + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "select by stackAsociation.data_group (LIKE comparison)", NULL); + + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-priority_order", 0, "order by release priority", false); + + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definerelgroup @@ -314,4 +351,5 @@ PXOPT_ADD_MODE("-definerelstack", "define a released stack", RELEASETOOL_MODE_DEFINERELSTACK, definerelstackArgs); PXOPT_ADD_MODE("-listrelstack", "list released stacks", RELEASETOOL_MODE_LISTRELSTACK, listrelstackArgs); + PXOPT_ADD_MODE("-summary", "list stackSummaryes for released stacks", RELEASETOOL_MODE_SUMMARY, summaryArgs); PXOPT_ADD_MODE("-definerelgroup", "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP, definerelgroupArgs); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.c (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.c (revision 36680) @@ -0,0 +1,423 @@ +/* + * remotetool.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "remotetool.h" + +// Create +static bool definebyqueryMode(pxConfig *config); + +// List +static bool listrunMode(pxConfig *config); +static bool listcomponentMode(pxConfig *config); + +// Update +static bool updaterunMode(pxConfig *config); +static bool updatepollMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); +static bool revertauthMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = remotetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(REMOTETOOL_MODE_LISTRUN, listrunMode); + MODECASE(REMOTETOOL_MODE_LISTCOMPONENT, listcomponentMode); + MODECASE(REMOTETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(REMOTETOOL_MODE_UPDATEPOLL, updatepollMode); + MODECASE(REMOTETOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(REMOTETOOL_MODE_REVERTAUTH, revertauthMode); + + default: + psAbort("invalid option (this should not happen)"); + } + psTrace("remotetool", 9, "Attempting to free config\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(EXIT_SUCCESS); + + FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +// Create + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // These are required + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // These are not + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + set_label = set_label ? set_label : label; + + // Get a list of the things we can insert into a new component + psString query = NULL; + if (!strcmp(stage,"chip")) { + query = pxDataGet("remotetool_definebyquery_chip.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (chipRun.label = '%s')", label); + } + } + else if (!strcmp(stage,"camera")) { + query = pxDataGet("remotetool_definebyquery_cam.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (camRun.label = '%s')", label); + } + } + else if (!strcmp(stage,"warp")) { + query = pxDataGet("remotetool_definebyquery_warp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (warpRun.label = '%s')", label); + } + } + else if (!strcmp(stage,"stack")) { + query = pxDataGet("remotetool_definebyquery_stack.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (stackRun.label = '%s')", label); + } + } + else { + psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); + return false; + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("remotetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "newremoteComponents", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + // Insert the top level run object + remoteRunRow *run = remoteRunRowAlloc(0, // remote_id + "new", // state + stage, + set_label, + path_base, + -1, // job_id + NULL, // last_poll + 0 // fault + ); + if (!run) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + return(true); + } + + if (!remoteRunInsertObject(config->dbh, run)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + psS64 remote_id = psDBLastInsertID(config->dbh); + + psArray *list = psArrayAllocEmpty(limit); + for (long i=0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + + remoteComponentRow *comp = remoteComponentRowAlloc(remote_id, + stage_id); + if (!comp) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object"); + return(true); + } + + if (!remoteComponentInsertObject(config->dbh, comp)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + return(true); + } + + psArrayAdd(list, list->n, comp); + psFree(comp); + + } + psFree(run); + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!remoteComponentPrintObjects(stdout, list, !simple)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(run); + return false; + } + + psFree(list); + psFree(output); + + return true; +} + +// List +static bool listrunMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-poll_begin", "last_poll", ">="); + PXOPT_COPY_TIME(config->args, where, "-poll_end", "last_poll", "<="); + + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + PXOPT_LOOKUP_S16(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + psArray *remRuns = NULL; + remRuns = psDBSelectRows(config->dbh, "remoteRun", where, limit); + + if (!ippdbPrintMetadatas(stdout, remRuns, "remoteRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(remRuns); + return false; + } + psFree(remRuns); + + return true; +} + +static bool listcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psArray *remComps = NULL; + remComps = psDBSelectRows(config->dbh, "remoteComponent", where, 0); + + if (!ippdbPrintMetadatas(stdout, remComps, "remoteComponent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(remComps); + return false; + } + psFree(remComps); + + return true; +} + +// Update + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // Wheres + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + + // Values to set + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, values, "-job_id", "job_id", "=="); + + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + +static bool updatepollMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + + psMetadata *where = psMetadataAlloc(); + // Wheres + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + psString query = NULL; + query = pxDataGet("remotetool_updatepoll.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement"); + psFree(whereClause); + return(false); + } + + if (!p_psDBRunQueryF(config->dbh, query,whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereClause); + return false; + } + psFree(query); + psFree(whereClause); + return(true); +} + + + + +static bool revertrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + // Value to set + PXOPT_COPY_S16(config->args, values, "-fault", "fault", "=="); + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + +static bool revertauthMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + + // required + PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth"); + + // Value to set + psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run"); + + long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values); + psFree(values); + psFree(where); + if (rows < 1) { + psError(PS_ERR_UNKNOWN,false, "no rows affected"); + return(false); + } + + return(true); +} + Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.h (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetool.h (revision 36680) @@ -0,0 +1,23 @@ +/* + * remotetool.h + */ + +#ifndef REMOTETOOL_H +#define REMOTETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + REMOTETOOL_MODE_NONE = 0x0, + REMOTETOOL_MODE_DEFINEBYQUERY, + REMOTETOOL_MODE_LISTRUN, + REMOTETOOL_MODE_LISTCOMPONENT, + REMOTETOOL_MODE_UPDATERUN, + REMOTETOOL_MODE_UPDATEPOLL, + REMOTETOOL_MODE_REVERTRUN, + REMOTETOOL_MODE_REVERTAUTH, +} remotetoolMode; + +pxConfig *remotetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // REMOTETOOL_H Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetoolConfig.c (revision 36680) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/remotetoolConfig.c (revision 36680) @@ -0,0 +1,124 @@ +/* + * remotetoolConfig.c + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pxtools.h" +#include "remotetool.h" + +#define ADD_OPT(TYPE,TARG,NAME,COMMENT,DEFAULT) psMetadataAdd##TYPE(TARG, PS_LIST_TAIL, NAME, 0, COMMENT, DEFAULT) + +pxConfig *remotetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + ADD_OPT(Str, definebyqueryArgs, "-label", "stage label to query", NULL); + ADD_OPT(Str, definebyqueryArgs, "-set_label", "remRun label to use", NULL); + ADD_OPT(Str, definebyqueryArgs, "-stage", "stage to query", NULL); + ADD_OPT(Str, definebyqueryArgs, "-path_base", "define base output location", NULL); + ADD_OPT(S16, definebyqueryArgs, "-limit", "limit to number of jobs to bundle",0); + ADD_OPT(Bool, definebyqueryArgs, "-pretend", "pretend to do this.", 0); + ADD_OPT(Bool, definebyqueryArgs,"-simple", "simple print format",0); + + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + ADD_OPT(S64, listrunArgs, "-remote_id", "run to list", 0); + ADD_OPT(Str, listrunArgs, "-state", "run state", NULL); + ADD_OPT(Str, listrunArgs, "-stage", "stage to return", NULL); + ADD_OPT(Str, listrunArgs, "-label", "remRun label to use", NULL); + + ADD_OPT(S64, listrunArgs, "-job_id", "job_id", 0); + // ADD_OPT(Str, listrunArgs, "-poll_begin", "last polled time begin", NULL); + // ADD_OPT(Str, listrunArgs, "-poll_end", "last polled time begin", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-poll_begin", 0, "last polled time begin", NULL); + psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-poll_end", 0, "last polled time end", NULL); + + ADD_OPT(S16, listrunArgs, "-fault", "fault code", 0); + + ADD_OPT(Bool, listrunArgs, "-simple", "simple print format", false); + ADD_OPT(S16, listrunArgs, "-limit", "limit to number of runs to display", 0); + + // -listcomponents + psMetadata *listcomponentArgs = psMetadataAlloc(); + ADD_OPT(S64, listcomponentArgs, "-remote_id", "run to list", 0); + ADD_OPT(S16, listcomponentArgs, "-limit", "limit to number of runs to display", 0); + ADD_OPT(Bool, listcomponentArgs, "-simple", "simple print format", false); + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + ADD_OPT(S64, updaterunArgs, "-remote_id", "run to update", 0); + // ADD_OPT(Str, updaterunArgs, "-label", "label to update", NULL); + ADD_OPT(Str, updaterunArgs, "-set_label", "remRun label to use", NULL); + ADD_OPT(Str, updaterunArgs, "-set_state", "remoteRun state to assign", NULL); + ADD_OPT(S64, updaterunArgs, "-job_id", "job_id to set", 0); + ADD_OPT(S16, updaterunArgs, "-fault", "fault code", 0); + // ADD_OPT(Bool, updaterunArgs, "-poll", "set poll date", NULL); + + // -updatepoll + psMetadata *updatepollArgs = psMetadataAlloc(); + ADD_OPT(S64, updatepollArgs, "-remote_id", "run to update", 0); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + ADD_OPT(S64, revertrunArgs, "-remote_id", "run to revert", 0); + ADD_OPT(Str, revertrunArgs, "-label", "remRun label to use", NULL); + ADD_OPT(S16, revertrunArgs, "-fault", "fault code", 0); + + // -revertauth + psMetadata *revertauthArgs = psMetadataAlloc(); + ADD_OPT(S64, revertauthArgs, "-remote_id", "run to revert", 0); + ADD_OPT(Str, revertauthArgs, "-label", "remRun label to use", NULL); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "", REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-listrun", "", REMOTETOOL_MODE_LISTRUN, listrunArgs); + PXOPT_ADD_MODE("-listcomponent", "", REMOTETOOL_MODE_LISTCOMPONENT, listcomponentArgs); + PXOPT_ADD_MODE("-updaterun", "", REMOTETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-updatepoll", "", REMOTETOOL_MODE_UPDATEPOLL, updatepollArgs); + PXOPT_ADD_MODE("-revertrun", "", REMOTETOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-revertauth", "", REMOTETOOL_MODE_REVERTAUTH, revertauthArgs); + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PXTOOLS_ERR_SYS, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} + Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.c (revision 36680) @@ -44,4 +44,6 @@ static bool tosummaryMode(pxConfig *config); static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); +static bool summaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); static bool pendingcleanupskyfileMode(pxConfig *config); @@ -84,4 +86,6 @@ MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(STACKTOOL_MODE_REVERTSUMMARY, revertsummaryMode); + MODECASE(STACKTOOL_MODE_SUMMARY, summaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode); @@ -748,4 +752,5 @@ PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "=="); @@ -1567,4 +1572,120 @@ } +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false); + + psString query = pxDataGet("stacktool_revertsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, sass_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return(false); + } + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %ld rows", numUpdated); + + psFree(query); + + return(true); +} + +static bool summaryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sass_id", "stackSummary.sass_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "stackAssociation.projection_cell", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackAssociation.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackAssociation.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); + pxAddLabelSearchArgs(config, where, "-data_group", "stackAssociation.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); +// PXOPT_COPY_STR(config->args, where, "-state", "stackRun.state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "LIKE"); +// PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); +// PXOPT_COPY_F64(config->args, where, "-mjd_obs_begin", "stackSumSkyfile.mjd_obs", ">="); +// PXOPT_COPY_F64(config->args, where, "-mjd_obs_end", "stackSumSkyfile.mjd_obs", "<="); +// PXOPT_COPY_S16(config->args, where, "-background_model","stackSumSkyfile.background_model", "=="); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("stacktool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("stacktool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "stackSummary", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + static bool pendingcleanuprunMode(pxConfig *config) { Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.h (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktool.h (revision 36680) @@ -38,4 +38,6 @@ STACKTOOL_MODE_TOSUMMARY, STACKTOOL_MODE_ADDSUMMARY, + STACKTOOL_MODE_REVERTSUMMARY, + STACKTOOL_MODE_SUMMARY, STACKTOOL_MODE_PENDINGCLEANUPRUN, STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktoolConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/stacktoolConfig.c (revision 36680) @@ -121,4 +121,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", 0); psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); @@ -264,4 +265,25 @@ psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -revertsummary + psMetadata *revertsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + + // -summary + psMetadata *summaryArgs = psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "search by projection cell ID", NULL); +// psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by stackAssociation data_group (LIKE comparison)", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); +// psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by stackRun dist_group (LIKE comparison)", NULL); + +// psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -pendingcleanuprun psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); @@ -317,12 +339,18 @@ PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); - PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); + PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); + PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + + PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); + PXOPT_ADD_MODE("-summary", "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs); PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert entry in the summary table", STACKTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", STACKTOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.c (revision 36680) @@ -50,4 +50,6 @@ static bool revertskycalresultMode(pxConfig *config); static bool updateskycalresultMode(pxConfig *config); +static bool exportskycalrunMode(pxConfig *config); +static bool importskycalrunMode(pxConfig *config); static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -88,4 +90,6 @@ MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode); MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultMode); + MODECASE(STATICSKYTOOL_MODE_EXPORTSKYCALRUN, exportskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_IMPORTSKYCALRUN, importskycalrunMode); default: psAbort("invalid option (this should not happen)"); @@ -895,75 +899,107 @@ bool importrunMode(pxConfig *config) { -# if (0) - unsigned int nFail; - - int numImportTables = 2; - - char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"}; + return false; +} + +bool exportskycalrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; PS_ASSERT_PTR_NON_NULL(config, NULL); - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - -#ifdef notdef - fprintf (stderr, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); -#endif - - if (!pxCheckImportVersion(config, input)) { - psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + // XXX unused PXOPT_LOOKUP_S64(det_id, config->args, "-sky_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); return false; } - psMetadataItem *item = psMetadataLookup (input, "stackRun"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - psMetadataItem *entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md); - stackRunInsertObject (config->dbh, stackRun); - - // fprintf (stdout, "---- stack run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - - for (int i = 0; i < numImportTables; i++) { - psMetadataItem *item = psMetadataLookup (input, tables[i]); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - switch (i) { - case 0: - for (int i = 0; i < item->data.list->n; i++) { - entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md); - stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - break; - - case 1: - for (int i = 0; i < item->data.list->n; i++) { - entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md); - stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - break; - } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; } - -# endif - return true; + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + + ExportTable tables [] = { + {"skycalRun", "staticskytool_export_skycalrun.sql"}, + {"skycalResult", "staticskytool_export_skycalresult.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "skycalRun")) { + if (!pxSetStateCleaned("skycalRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + } + + fclose (f); + + return true; +} + +bool importskycalrunMode(pxConfig *config) +{ + return false; } @@ -986,37 +1022,4 @@ } - -# if (0) - // now we need to loop over all requested filters and check that each is found - // XXX is this needed? haven't we required that we only match the requested filters - // and that N unique filters are found? - - psVector *found = psVectorAlloc(inputs->n, PS_TYPE_U8); - psVectorInit (found, 0); - - psMetadataItem *filter = NULL; - psListIterator *iter = psListIteratorAlloc (filters->data.list, PS_LIST_HEAD, false); - while ((filter = psListGetAndIncrement(iter))) { - - bool foundOne = false; - - psAssert (filter->type == PS_DATA_STR, "filter is not a string?"); - for (int j = 0; !foundOne && (j < inputs->n); j++) { - if (found->data.U8[j]) continue; - psMetadata *inputRow = inputs->data[j]; // Row from select - - // pull out the skycell_id, tess_id, filter - psString inFilter = psMetadataLookupStr(&status, row, "filter"); - psAssert (status); - - found->data.U8[j] = true; - foundOne = true; - } - if (!foundOne) { - // this required filter was not found in the inputs, skip the entry - skip(); - } - } -# endif static bool defineskycalrunMode(pxConfig *config) @@ -1042,8 +1045,8 @@ PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); pxAddLabelSearchArgs(config, whereMD, "-select_label", "staticskyRun.label", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.h =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.h (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytool.h (revision 36680) @@ -43,4 +43,6 @@ STATICSKYTOOL_MODE_SKYCALRESULT, STATICSKYTOOL_MODE_REVERTSKYCALRESULT, + STATICSKYTOOL_MODE_EXPORTSKYCALRUN, + STATICSKYTOOL_MODE_IMPORTSKYCALRUN, } staticskytoolMode; Index: branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytoolConfig.c (revision 35860) +++ branches/eam_branches/ps2-tc3-20130727/ippTools/src/staticskytoolConfig.c (revision 36680) @@ -269,4 +269,16 @@ psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -exportskycalrun + psMetadata *exportskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "export this skycal ID (required)", 0); + psMetadataAddStr(exportskycalrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportskycalrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + + // -importrun + psMetadata *importskycalrunArgs = psMetadataAlloc(); + psMetadataAddStr(importskycalrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + psFree(now); @@ -291,4 +303,6 @@ PXOPT_ADD_MODE("-updateskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs); PXOPT_ADD_MODE("-skycalresult", "Get result of skycal run", STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs); + PXOPT_ADD_MODE("-exportskycalrun", "Export skycal run", STATICSKYTOOL_MODE_EXPORTSKYCALRUN, exportskycalrunArgs); + PXOPT_ADD_MODE("-importskycalrun", "Import skycal run", STATICSKYTOOL_MODE_IMPORTSKYCALRUN, importskycalrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {