Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 32959) +++ trunk/ippTools/share/Makefile.am (revision 32960) @@ -385,4 +385,6 @@ staticskytool_definebyquery_select_by_dg.sql \ staticskytool_definebyquery_inputs.sql \ + staticskytool_defineskycalrun.sql \ + staticskytool_pendingskycalrun.sql \ staticskytool_export_run.sql \ staticskytool_export_input.sql \ Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 32959) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 32960) @@ -2035,4 +2035,48 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE skycalRun ( + skycal_id BIGINT AUTO_INCREMENT, -- unique identifier + sky_id BIGINT, + stack_id BIGINT, + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(skycal_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(sky_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of sky calibration analysis +CREATE TABLE skycalResult ( + skycal_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + dtime_astrom FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + n_astrom INT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + fault SMALLINT NOT NULL, + PRIMARY KEY(skycal_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + -- This comment line is here to avoid empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: trunk/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_drop_tables.sql (revision 32959) +++ trunk/ippTools/share/pxadmin_drop_tables.sql (revision 32960) @@ -86,4 +86,5 @@ DROP TABLE IF EXISTS publishDone; DROP TABLE IF EXISTS pstampDependent; +DROP TABLE IF EXISTS pstampFile; DROP TABLE IF EXISTS staticskyInput; DROP TABLE IF EXISTS staticskyResult; @@ -105,4 +106,6 @@ DROP TABLE IF EXISTS vpProcessedCell; DROP TABLE IF EXISTS vpRun; +DROP TABLE IF EXISTS skycalRun; +DROP TABLE IF EXISTS skycalResult; SET FOREIGN_KEY_CHECKS=1 Index: trunk/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- trunk/ippTools/share/staticskytool_defineskycalrun.sql (revision 32960) +++ trunk/ippTools/share/staticskytool_defineskycalrun.sql (revision 32960) @@ -0,0 +1,16 @@ +SELECT + sky_id, + stack_id, + staticskyRun.workdir, + staticskyRun.label, + staticskyRun.data_group, + staticskyRun.dist_group +FROM staticskyRun + JOIN staticskyResult USING(sky_id) + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + LEFT JOIN skycalRun USING(sky_id, stack_id) +WHERE staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + Index: trunk/ippTools/share/staticskytool_pendingskycalrun.sql =================================================================== --- trunk/ippTools/share/staticskytool_pendingskycalrun.sql (revision 32960) +++ trunk/ippTools/share/staticskytool_pendingskycalrun.sql (revision 32960) @@ -0,0 +1,21 @@ +SELECT + skycalRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + staticskyResult.path_base, + rawExp.camera, + staticskyResult.num_inputs = 1 AS singlefilter +FROM skycalRun +JOIN staticskyRun USING (sky_id) +JOIN staticskyResult USING(sky_id) +JOIN stackRun USING (stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING (chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN skycalResult USING(skycal_id) +WHERE + (skycalRun.state = 'new' AND skycalResult.skycal_id IS NULL) Index: trunk/ippTools/src/staticskytool.c =================================================================== --- trunk/ippTools/src/staticskytool.c (revision 32959) +++ trunk/ippTools/src/staticskytool.c (revision 32960) @@ -42,4 +42,8 @@ static bool importrunMode(pxConfig *config); +static bool defineskycalrunMode(pxConfig *config); +static bool updateskycalrunMode(pxConfig *config); +static bool pendingskycalrunMode(pxConfig *config); + static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -72,4 +76,7 @@ MODECASE(STATICSKYTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(STATICSKYTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunMode); default: psAbort("invalid option (this should not happen)"); @@ -97,10 +104,8 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); @@ -111,4 +116,5 @@ psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); @@ -978,2 +984,238 @@ # endif +static bool defineskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + // none required. We get workdir, etc from staticskyRun if not provided + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id", "staticskyRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("staticskytool_defineskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + if (!rerun) { + if (label) { + psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND skycal_id IS NULL"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(output)) { + psWarning("staticskytool: no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "skycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 sky_id = psMetadataLookupS64(&status, row, "sky_id"); + psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id"); + psString sky_workdir = psMetadataLookupStr(&status, row, "workdir"); + psString sky_label = psMetadataLookupStr(&status, row, "label"); + psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + + // create a staticskyRun + if (!skycalRunInsert(config->dbh, + 0x0, // skycal_id + sky_id, + stack_id, + "new", // state + workdir ? workdir : sky_workdir, + label ? label : sky_label, + data_group ? data_group : sky_data_group, + dist_group, + reduction, + registered, + note + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + } + psFree(output); + return true; +} + +static bool updateskycalrunMode(pxConfig *config) +{ +#ifdef notyet + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE staticskyRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false); + psFree(query); + psFree(where); + + return result; +#endif + return false; +} +static bool pendingskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_pendingskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + +// psStringAppend(&query, "\n GROUP by skycal_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingskycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} Index: trunk/ippTools/src/staticskytool.h =================================================================== --- trunk/ippTools/src/staticskytool.h (revision 32959) +++ trunk/ippTools/src/staticskytool.h (revision 32960) @@ -34,5 +34,13 @@ STATICSKYTOOL_MODE_UPDATERESULT, STATICSKYTOOL_MODE_EXPORTRUN, - STATICSKYTOOL_MODE_IMPORTRUN + STATICSKYTOOL_MODE_IMPORTRUN, + + STATICSKYTOOL_MODE_DEFINESKYCALRUN, + STATICSKYTOOL_MODE_PENDINGSKYCALRUN, + STATICSKYTOOL_MODE_UPDATESKYCALRUN, + STATICSKYTOOL_MODE_ADDSKYCALRESULT, + STATICSKYTOOL_MODE_UPDATESKYCALRESULT, + STATICSKYTOOL_MODE_SKYCALRESULT, + STATICSKYTOOL_MODE_REVERTSKYCALRESULT, } staticskytoolMode; Index: trunk/ippTools/src/staticskytoolConfig.c =================================================================== --- trunk/ippTools/src/staticskytoolConfig.c (revision 32959) +++ trunk/ippTools/src/staticskytoolConfig.c (revision 32960) @@ -48,4 +48,5 @@ // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); @@ -138,5 +139,46 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - + + // -defineskycalrun + psMetadata *defineskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_sky_id", 0, "search for sky_id", 0); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(defineskycalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateskycalrun + psMetadata *updateskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -pendingskycalrun + psMetadata *pendingskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddU64(pendingskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -155,4 +197,7 @@ PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-defineskycalrun", "Define a new skycalrun", STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunArgs); + PXOPT_ADD_MODE("-updateskycalrun", "Update a skycalrun", STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunArgs); + PXOPT_ADD_MODE("-pendingskycalrun", "Get skcal runs to do", STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {