Index: branches/eam_branches/ipp-20140206/ippTools/src/detselect.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/detselect.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/detselect.c (revision 36627) @@ -29,4 +29,6 @@ static bool searchMode(pxConfig *config); static bool selectMode(pxConfig *config); +static bool showMode(pxConfig *config); + # define MODECASE(caseName, func) \ @@ -50,4 +52,5 @@ MODECASE(DETSELECT_MODE_SEARCH, searchMode); MODECASE(DETSELECT_MODE_SELECT, selectMode); + MODECASE(DETSELECT_MODE_SHOW, showMode); default: psAbort("invalid option (this should not happen)"); @@ -241,2 +244,64 @@ return true; } + +static bool showMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-det_type", "det_type", "=="); + PXOPT_COPY_STR(config->args, where, "-type", "det_type", "=="); + + psString query = pxDataGet("detselect_show.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // we choose the single detrend image which matches all criteria and has + // the latest insertion date + + psStringAppend(&query, " ORDER BY registered DESC, iteration DESC"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + // XXX check psError here + psError(PS_ERR_UNKNOWN, false, "no detRun rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "detRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + return(true); +} Index: branches/eam_branches/ipp-20140206/ippTools/src/detselect.h =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/detselect.h (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/detselect.h (revision 36627) @@ -27,4 +27,5 @@ DETSELECT_MODE_SEARCH, DETSELECT_MODE_SELECT, + DETSELECT_MODE_SHOW } detselectMode; Index: branches/eam_branches/ipp-20140206/ippTools/src/detselectConfig.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/detselectConfig.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/detselectConfig.c (revision 36627) @@ -65,4 +65,12 @@ psMetadataAddStr(selectArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddBool(selectArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -show + psMetadata *showArgs = psMetadataAlloc(); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-det_type", 0, "search by detrend type", NULL); + psMetadataAddStr(showArgs, PS_LIST_TAIL, "-type", 0, "search by detrend type (alias for -det_type)", NULL); + psMetadataAddBool(showArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadata *argSets = psMetadataAlloc(); @@ -71,4 +79,5 @@ PXOPT_ADD_MODE("-search", "search for an appropriate detrend", DETSELECT_MODE_SEARCH, searchArgs); PXOPT_ADD_MODE("-select", "retreive detrend information", DETSELECT_MODE_SELECT, selectArgs); + PXOPT_ADD_MODE("-show", "show all active detrends of this type with constraints", DETSELECT_MODE_SHOW, showArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ipp-20140206/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/difftool.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/difftool.c (revision 36627) @@ -1349,4 +1349,12 @@ if (!mdok) { psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psString warpDataGroup = psMetadataLookupStr(&mdok, row, "warpDataGroup"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warpDataGroup not found"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1391,5 +1399,5 @@ workdir, label, - data_group ? data_group : label, + data_group ? data_group : warpDataGroup, dist_group, reduction, Index: branches/eam_branches/ipp-20140206/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/disttool.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/disttool.c (revision 36627) @@ -413,4 +413,21 @@ magic = false; + } else if (!strcmp(stage, "ff")) { + runJoinStr = "fullForceRun.ff_id"; + query = pxDataGet("disttool_definebyquery_ff.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return(false); + } + + if (label) { + psStringAppend(&query, " AND (fullForceRun.label = '%s') ", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fullForceRun.dist_group = '%s') ", dist_group); + } + + magic = false; } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); Index: branches/eam_branches/ipp-20140206/ippTools/src/fftool.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/fftool.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/fftool.c (revision 36627) @@ -44,4 +44,7 @@ static bool updatesummaryMode(pxConfig *config); static bool summaryMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); +static bool summaryMode(pxConfig *config); static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -77,4 +80,6 @@ MODECASE(FFTOOL_MODE_UPDATESUMMARY, updatesummaryMode); MODECASE(FFTOOL_MODE_SUMMARY, summaryMode); + MODECASE(FFTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(FFTOOL_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -134,7 +139,7 @@ psMetadata *warpWhereMD = psMetadataAlloc(); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); - PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "fullForceRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","fullForceRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "fullForceRun.warp_id", "=="); if (!psListLength(warpWhereMD->list)) { psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); @@ -150,6 +155,6 @@ } - PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); - pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "warpRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "fullForceRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "fullForceRun.filter", "LIKE"); if (!pxskycellAddWhere(config, warpWhereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -175,5 +180,5 @@ psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); psStringAppend(&where, "\nAND %s", whereClause); - psStringAppend(&select, where); + psStringAppend(&select, " %s", where); psFree(whereClause); psFree(skycalWhereMD); @@ -504,4 +509,5 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); pxskycellAddWhere(config, where); @@ -864,4 +870,5 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); pxskycellAddWhere(config, where); @@ -933,2 +940,176 @@ return true; } + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +// PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + ExportTable tables [] = { + {"fullForceRun", "fftool_export_run.sql"}, + {"fullForceInput", "fftool_export_input.sql"}, + {"fullForceResult", "fftool_export_result.sql"}, + {"fullForceSummary", "fftool_export_summary.sql"}, + }; + + int numTables = sizeof(tables)/sizeof(tables[0]); + + for (int i=0; i < numTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + int numImportTables = 3; + + char tables[3] [80] = {"fullForceInput", "fullForceResult", "fullForceSummary"}; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + +#ifdef notdef + fprintf (stderr, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + + psMetadataItem *item = psMetadataLookup (input, "fullForceRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceRunRow *fullForceRun = fullForceRunObjectFromMetadata (entry->data.md); + fullForceRunInsertObject (config->dbh, fullForceRun); + + // fprintf (stdout, "---- warp run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + for (int i = 0; i < numImportTables; i++) { + item = psMetadataLookup (input, tables[i]); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + switch (i) { + case 0: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceInputRow *fullForceInput = fullForceInputObjectFromMetadata (entry->data.md); + fullForceInputInsertObject (config->dbh, fullForceInput); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 1: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceResultRow *fullForceResult = fullForceResultObjectFromMetadata (entry->data.md); + fullForceResultInsertObject (config->dbh, fullForceResult); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + + case 2: + for (int i = 0; i < item->data.list->n; i++) { + entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + fullForceSummaryRow *fullForceSummary = fullForceSummaryObjectFromMetadata (entry->data.md); + fullForceSummaryInsertObject (config->dbh, fullForceSummary); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + break; + } + } + return true; +} Index: branches/eam_branches/ipp-20140206/ippTools/src/fftool.h =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/fftool.h (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/fftool.h (revision 36627) @@ -36,5 +36,7 @@ FFTOOL_MODE_REVERTSUMMARY, FFTOOL_MODE_UPDATESUMMARY, - FFTOOL_MODE_SUMMARY + FFTOOL_MODE_SUMMARY, + FFTOOL_MODE_EXPORTRUN, + FFTOOL_MODE_IMPORTRUN, } fftoolMode; Index: branches/eam_branches/ipp-20140206/ippTools/src/fftoolConfig.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/fftoolConfig.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/fftoolConfig.c (revision 36627) @@ -186,4 +186,17 @@ pxskycellAddArguments(revertsummaryArgs); + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-ff_id", 0, "export this full force ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); +// psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psFree(now); @@ -203,4 +216,6 @@ PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); PXOPT_ADD_MODE("-summary", "list summary results", FFTOOL_MODE_SUMMARY, summaryArgs); + PXOPT_ADD_MODE("-exportrun", "list summary results", FFTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "list summary results", FFTOOL_MODE_IMPORTRUN, importrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ipp-20140206/ippTools/src/laptool.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/laptool.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/laptool.c (revision 36627) @@ -1170,5 +1170,7 @@ // we only want projection cells which do not already of an entry psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n"); - psStringAppend(&query, "\nORDER by projection_cell\n"); + // Projection cell's don't have a convienient order. Queue runs in the order of lap_id + // psStringAppend(&query, "\nORDER by projection_cell\n"); + psStringAppend(&query, "\nORDER by lap_id_0\n"); if (limit) { @@ -1289,4 +1291,5 @@ psFree(whereClause); } + psStringAppend(&query, "\nGROUP by projection_cell ORDER by MIN(lap_id)"); if (limit) { psString limitString = psDBGenerateLimitSQL(limit); Index: branches/eam_branches/ipp-20140206/ippTools/src/remotetool.c =================================================================== --- branches/eam_branches/ipp-20140206/ippTools/src/remotetool.c (revision 36536) +++ branches/eam_branches/ipp-20140206/ippTools/src/remotetool.c (revision 36627) @@ -149,5 +149,5 @@ psFree(limitString); } - if (!p_psDBRunQueryF(config->dbh, query)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -313,5 +313,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); psMetadata *where = psMetadataAlloc(); @@ -341,5 +341,5 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); + // PXOPT_LOOKUP_S64(remote_id, config->args, "-remote_id", true, false); psMetadata *where = psMetadataAlloc();