Index: /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/share/Makefile.am (revision 42524) @@ -523,4 +523,6 @@ fftool_definebyquery.sql \ fftool_definebyquery_select_warps.sql \ + fftool_defineforstacks.sql \ + fftool_defineforstacks_select_warps.sql \ fftool_result.sql \ fftool_revert.sql \ Index: /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks.sql (revision 42524) +++ /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks.sql (revision 42524) @@ -0,0 +1,14 @@ +SELECT + skycalRun.skycal_id, + skycalResult.path_base, + skycalRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter +FROM skycalRun + JOIN skycalResult USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) + -- join hook %s +WHERE + skycalRun.state = 'full' AND skycalResult.quality = 0 Index: /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks_select_warps.sql =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks_select_warps.sql (revision 42524) +++ /branches/eam_branches/ipp-20230313/ippTools/share/fftool_defineforstacks_select_warps.sql (revision 42524) @@ -0,0 +1,5 @@ +SELECT warp_id +FROM stackInputSkyfile + JOIN stackRun USING (stack_id) + JOIN skycalRun USING (stack_id) +WHERE skycal_id = '%d' Index: /branches/eam_branches/ipp-20230313/ippTools/src/fftool.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/fftool.c (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/src/fftool.c (revision 42524) @@ -33,4 +33,5 @@ static bool definebyqueryMode(pxConfig *config); +static bool defineforstacksMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todoMode(pxConfig *config); @@ -68,4 +69,5 @@ switch (config->mode) { MODECASE(FFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(FFTOOL_MODE_DEFINEFORSTACKS, defineforstacksMode); MODECASE(FFTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(FFTOOL_MODE_TODO, todoMode); @@ -337,4 +339,216 @@ } +static bool defineforstacksMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + // optional + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + + PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); + + psMetadata *skycalWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); + + if (!psListLength(skycalWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, skycalWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString select = pxDataGet("fftool_defineforstacks.sql"); + if (!select) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(skycalWhereMD); + return false; + } + + psString where = NULL; + psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); + psStringAppend(&where, "\nAND %s", whereClause); + psStringAppend(&select, " %s", where); + psFree(whereClause); + psFree(skycalWhereMD); + + psString joinHook = NULL; + if (!rerun) { + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = skycalRun.skycal_id"); + psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); + psStringAppend(&select, "\nAND ff_id IS NULL"); + } + + if (!p_psDBRunQueryF(config->dbh, select, joinHook)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(select); + return false; + } + psFree(select); + psFree(joinHook); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + psFree(where); + return false; + } + if (!psArrayLength(output)) { + psWarning("fftool: no rows found"); + psFree(output); + psFree(where); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + psFree(where); + return false; + } + psFree(output); + psFree(where); + return true; + } + + psString warpQueryTemplate = pxDataGet("fftool_defineforstacks_select_warps.sql"); + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id"); + + psString path_base = NULL; + if (sources_path_base) { + path_base = sources_path_base; + } else { + path_base = psMetadataLookupStr(&status, row, "path_base"); + psAssert(status, "failed to find skycal path_base?"); + } + + psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group"); + + psString query = NULL; + psStringAppend(&query, warpQueryTemplate, skycal_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // Find the warps for this skycell and filter combination + psArray *warpOutput = p_psDBFetchResult(config->dbh); + if (!warpOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(warpOutput)) { + // no warps for this skycal. Suprise? + psFree(warpOutput); + psFree(query); + continue; + } + + // create a staticskyRun + if (!fullForceRunInsert(config->dbh, + 0x0, // ff_id + skycal_id, + path_base, + "new", // state + workdir, + label, + data_group ? data_group : (skycal_data_group ? skycal_data_group : label), + dist_group, + note, + reduction, + NULL // registered + ) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psS64 ff_id = psDBLastInsertID(config->dbh); + + for (int j = 0; j < warpOutput->n; j++) { + psMetadata *warpRow = warpOutput->data[j]; + psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); + + if (!fullForceInputInsert(config->dbh, + ff_id, + warp_id) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + psFree(warpOutput); + } + psFree(output); + + return true; +} + static bool updaterunMode(pxConfig *config) { Index: /branches/eam_branches/ipp-20230313/ippTools/src/fftool.h =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/fftool.h (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/src/fftool.h (revision 42524) @@ -26,4 +26,5 @@ FFTOOL_MODE_NONE = 0x0, FFTOOL_MODE_DEFINEBYQUERY, + FFTOOL_MODE_DEFINEFORSTACKS, FFTOOL_MODE_UPDATERUN, FFTOOL_MODE_TODO, Index: /branches/eam_branches/ipp-20230313/ippTools/src/fftoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/fftoolConfig.c (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/src/fftoolConfig.c (revision 42524) @@ -71,4 +71,27 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -defineforstacks + psMetadata *defineforstacksArgs = psMetadataAlloc(); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_label", PS_META_DUPLICATE_OK, "search by skycal label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_data_group", PS_META_DUPLICATE_OK, "search by skycal data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddS64(defineforstacksArgs, PS_LIST_TAIL, "-select_skycal_id", 0, "search by skycal ID", 0); + + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddF32(defineforstacksArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "mimimum good_frac in warp", 0.0); + + pxskycellAddArguments(defineforstacksArgs); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_sources_path_base", 0, "define workdir", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(defineforstacksArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(defineforstacksArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun @@ -211,4 +234,5 @@ PXOPT_ADD_MODE("-definebyquery", "Define a new full force run", FFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-defineforstacks", "Define a new full force runs for inputs to stacks", FFTOOL_MODE_DEFINEFORSTACKS, defineforstacksArgs); PXOPT_ADD_MODE("-updaterun", "Update a run", FFTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-todo", "Get runs to do", FFTOOL_MODE_TODO, todoArgs); Index: /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexp.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexp.c (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexp.c (revision 42524) @@ -77,4 +77,5 @@ PXOPT_LOOKUP_S32(timeout, config->args, "-timeout", false, false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(ignore_errors, config->args, "-ignore-errors", false); // find last fileset/exp_name (if we have one) @@ -180,4 +181,5 @@ } + // XXX for test, make this not a temporary table: // create a temporry table { @@ -197,5 +199,8 @@ { - char *query = "INSERT INTO incoming (exp_name, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + char *query_notice = "INSERT INTO incoming (exp_name, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + char *query_ignore = "INSERT IGNORE INTO incoming (exp_name, camera, telescope, dateobs, exp_type, uri) VALUES (?, ?, ?, ?, ?, ?)"; + + char *query = ignore_errors ? query_ignore : query_notice; long inserted = p_psDBRunQueryPrepared(config->dbh, newSummitExps, query); @@ -209,4 +214,5 @@ return false; } + fprintf (stderr, "inserted: %d\n", (int) inserted); // sanity check that we actually inserted something if (inserted == 0) { @@ -222,4 +228,67 @@ psFree(newSummitExps); + + // TEST + if (0) { + char *query = "select * from incomingTest"; + p_psDBRunQuery(config->dbh, query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "test", TRUE)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + exit (2); + } + + // debug this query + if (0) { + char *query = + "DESCRIBE INSERT INTO summitExp" + " SELECT" + " NULL," // summit_id + " incomingTest.*," + " NULL," // imfiles + " 0," // fault + " NULL" // epoch + " FROM incomingTest" + " LEFT JOIN summitExp" + " USING(exp_name, camera, telescope)" + " WHERE" + " summitExp.exp_name is NULL" + " AND summitExp.camera is NULL" + " AND summitExp.telescope is NULL"; + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "test", TRUE)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + // exit (1); // add new exps to summitExp @@ -239,5 +308,9 @@ " summitExp.exp_name is NULL" " AND summitExp.camera is NULL" - " AND summitExp.telescope is NULL"; + " AND summitExp.telescope is NULL"; + + // MAJOR hack to avoid excess long inserts + // include this line to limit the incomping entries + // " AND incoming.dateobs > '2023-01-01'"; if (!p_psDBRunQuery(config->dbh, query)) { Index: /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexpConfig.c =================================================================== --- /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexpConfig.c (revision 42523) +++ /branches/eam_branches/ipp-20230313/ippTools/src/pzgetexpConfig.c (revision 42524) @@ -55,4 +55,6 @@ psMetadataAddBool(args, PS_LIST_TAIL, "-all", 0, "download ALL filesets", 0); + psMetadataAddBool(args, PS_LIST_TAIL, "-ignore-errors", 0, + "do not fail if one of the entries raises a database error", 0);