Index: branches/simmosaic_branches/ippTools/src/warptool.c =================================================================== --- branches/simmosaic_branches/ippTools/src/warptool.c (revision 24860) +++ branches/simmosaic_branches/ippTools/src/warptool.c (revision 27839) @@ -53,8 +53,12 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool setskyfiletoupdateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -102,8 +106,12 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); MODECASE(WARPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); + MODECASE(WARPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode); default: @@ -137,4 +145,7 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) @@ -158,4 +169,6 @@ "dirty", // workdir_state label, + data_group ? data_group : label, + dist_group, dvodb, tess_id, @@ -163,5 +176,6 @@ end_stage, registered, - 0 // magicked + 0, // magicked + note ); if (!warpRun) { @@ -234,13 +248,13 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); // define using fake label if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -249,8 +263,11 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -266,5 +283,5 @@ psString query = pxDataGet("warptool_definebyquery.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); return false; @@ -338,8 +355,11 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, reduction ? reduction : row->reduction, - end_stage ? end_stage : row->end_stage)) + end_stage ? end_stage : row->end_stage, + note)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -367,45 +387,33 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group","=="); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group","=="); + pxAddLabelSearchArgs(config, where, "-label", "warpRun.label", "=="); + + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { psFree(where); - where = NULL; - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); - psFree(where); - return false; - } - - if (state) { - // set warpRun.state to state - if (!pxwarpRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxwarpRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpRun JOIN warpSkyfile USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true); + + psFree(query); psFree(where); - return true; -} - + return result; +} static bool expMode(pxConfig *config) @@ -423,5 +431,5 @@ psString query = pxDataGet("warptool_exp.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -497,5 +505,5 @@ psString query = pxDataGet("warptool_imfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -571,5 +579,5 @@ psString query = pxDataGet("warptool_tooverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -680,5 +688,4 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); @@ -687,5 +694,5 @@ && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); return false; } @@ -695,5 +702,5 @@ psString query = pxDataGet("warptool_revertoverlap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -827,5 +834,5 @@ psString query = pxDataGet("warptool_scmap.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -900,11 +907,12 @@ psString query = pxDataGet("warptool_towarped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); } @@ -912,15 +920,29 @@ // treat limit == 0 as "no limit" + psString limitString = psStringCopy("\n"); if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); + // We apply the limit to both sides of the UNION + // to avoid slow queries and to the query itself + // to satisfy the user's requested limit + psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + } + +#define TWO_WHERES +#ifdef TWO_WHERES + if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } +#else + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } +#endif + psFree(limitString); + psFree(whereStr); psFree(query); @@ -981,5 +1003,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); // default values @@ -1044,5 +1066,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1120,5 +1142,5 @@ psString query = pxDataGet("warptool_finished_run_select.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); return false; } @@ -1190,8 +1212,18 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1201,5 +1233,17 @@ psString query = pxDataGet("warptool_warped.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } @@ -1207,6 +1251,17 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } } psFree(where); @@ -1270,60 +1325,56 @@ PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); - // PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); - return false; - } - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(where); - return false; - } - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("warptool_revertwarped_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("warptool_revertwarped_delete.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString query_updated = pxDataGet("warptool_revertwarped_updated.sql"); + if (!query_updated) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query_updated, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted); - psFree(where); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // fix any faulted warpSkyfiles in data_state 'update' + + if (!p_psDBRunQuery(config->dbh, query_updated)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query_updated); + return false; + } + psFree(query_updated); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated); return true; @@ -1412,5 +1463,5 @@ psString query = pxDataGet("warptool_pendingcleanuprun.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1475,5 +1526,5 @@ psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1536,5 +1587,5 @@ psString query = pxDataGet("warptool_donecleanup.sql"); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1624,7 +1675,20 @@ } - // note only updates if warpRun.state = run_state - // XXX note that we have removed this constraint for now - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) { + psString set_magicked_skyfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (!strcmp(data_state, "full")) { + // magicked is only an argument for for -tofullskyfile + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + if (magicked) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); + psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); + } + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is currently nonzero set it to -1 + // Set warpRun.magicked when the first skyfile is cleaned + psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1636,7 +1700,8 @@ } psFree(query); + psFree(set_magicked_skyfile); query = pxDataGet("warptool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) { + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) { // rollback if (!psDBRollback(config->dbh)) { @@ -1646,4 +1711,5 @@ return false; } + psFree(set_magicked_run); if (!psDBCommit(config->dbh)) { @@ -1665,4 +1731,8 @@ { return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); } @@ -1675,12 +1745,30 @@ PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged","goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; @@ -1707,4 +1795,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); @@ -1722,5 +1814,5 @@ psString query = pxDataGet(tables[i].sqlFilename); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } @@ -1799,6 +1891,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "warpRun"); @@ -1863,2 +1962,234 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("warptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "warpRun.dist_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("warptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpRun")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + if (where2) { + if (psListLength(where->list)) { + psStringAppend(&query, " %s", where2); + } else { + psStringAppend(&query, " WHERE 1 %s", where2); + } + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, warp_id DESC"); + } + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned warpSkyfile to be updated +static bool setskyfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("warptool_setskyfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , warpRun.label = '%s'", label); + } + + if (skycell_id) { + psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +}