Index: trunk/ippTools/scripts/test_create_detrend.sql =================================================================== --- trunk/ippTools/scripts/test_create_detrend.sql (revision 23229) +++ trunk/ippTools/scripts/test_create_detrend.sql (revision 23229) @@ -0,0 +1,135 @@ +-- disable constraints + +DROP TABLE IF EXISTS detRun; +DROP TABLE IF EXISTS detProcessedImfile; +DROP TABLE IF EXISTS detNormalizedStatImfile; +DROP TABLE IF EXISTS detResidImfile; + +CREATE TABLE detRun ( + det_id BIGINT AUTO_INCREMENT, + iteration INT, + det_type VARCHAR(64), + mode VARCHAR(64), + state VARCHAR(64), + filelevel VARCHAR(64), + workdir VARCHAR(255), + camera VARCHAR(64), + telescope VARCHAR(64), + exp_type VARCHAR(64), + reduction VARCHAR(64), + filter VARCHAR(64), + airmass_min FLOAT, + airmass_max FLOAT, + exp_time_min FLOAT, + exp_time_max FLOAT, + ccd_temp_min FLOAT, + ccd_temp_max FLOAT, + posang_min DOUBLE, + posang_max DOUBLE, + registered DATETIME, + time_begin DATETIME, + time_end DATETIME, + use_begin DATETIME, + use_end DATETIME, + solang_min FLOAT, + solang_max FLOAT, + label VARCHAR(64), + ref_det_id BIGINT, + ref_iter INT, + -- parent INT, :: dropping this + PRIMARY KEY(det_id), + KEY(det_id), + KEY(iteration), + KEY(det_type), + KEY(mode), + KEY(state), + KEY(label), + -- KEY(parent), :: dropping this + INDEX(det_id, iteration)) +ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detProcessedImfile ( + det_id BIGINT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, exp_id, class_id), + KEY(fault), + INDEX(det_id, class_id), + INDEX(det_id, exp_id) + -- FOREIGN KEY (det_id, exp_id) + -- REFERENCES detInputExp(det_id, exp_id), + -- FOREIGN KEY (exp_id, class_id) + -- REFERENCES rawImfile(exp_id, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detNormalizedStatImfile ( + det_id BIGINT, + iteration INT, + class_id VARCHAR(64), + norm FLOAT, + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, class_id), + KEY(fault) + -- FOREIGN KEY (det_id, iteration) + -- REFERENCES detInputExp(det_id, iteration), + -- FOREIGN KEY (det_id, iteration, class_id) + -- REFERENCES detStackedImfile(det_id, iteration, class_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE detResidImfile ( + det_id BIGINT, + iteration INT, + ref_det_id BIGINT, + ref_iter INT, + exp_id BIGINT, + class_id VARCHAR(64), + uri VARCHAR(255), + recipe VARCHAR(64), + bg DOUBLE, + bg_stdev DOUBLE, + bg_mean_stdev DOUBLE, + bg_skewness DOUBLE, + bg_kurtosis DOUBLE, + bin_stdev DOUBLE, + fringe_0 DOUBLE, + fringe_1 DOUBLE, + fringe_2 DOUBLE, + fringe_resid_0 DOUBLE, + fringe_resid_1 DOUBLE, + fringe_resid_2 DOUBLE, + user_1 DOUBLE, + user_2 DOUBLE, + user_3 DOUBLE, + user_4 DOUBLE, + user_5 DOUBLE, + path_base VARCHAR(255), + data_state VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(det_id, iteration, exp_id, class_id), + KEY(fault), + INDEX(det_id, iteration, exp_id) + -- FOREIGN KEY (det_id, iteration, exp_id) + -- REFERENCES detInputExp(det_id, iteration, exp_id), + -- FOREIGN KEY (det_id, exp_id, class_id) + -- REFERENCES detProcessedImfile(det_id, exp_id, class_id), + -- FOREIGN KEY (ref_det_id, ref_iter) + -- REFERENCES detNormalizedExp(det_id, iteration) +) ENGINE=innodb DEFAULT CHARSET=latin1; + Index: trunk/ippTools/scripts/test_insert_detrend.sql =================================================================== --- trunk/ippTools/scripts/test_insert_detrend.sql (revision 23229) +++ trunk/ippTools/scripts/test_insert_detrend.sql (revision 23229) @@ -0,0 +1,47 @@ +-- create a fake detrun, add fake detProcessedImfile entries, etc + +insert into + detRun (det_id, det_type, iteration, camera, workdir, state, mode) + values (1, 'flat', 0, 'GPC1', 'testdir', 'run', 'master'); + +insert into + detProcessedImfile (det_id, class_id) + values (1, 'ccd00'); + +insert into + detProcessedImfile (det_id, class_id) + values (1, 'ccd01'); + +insert into + detResidImfile (det_id, iteration, class_id) + values (1, 0, 'ccd00'); + +insert into + detResidImfile (det_id, iteration, class_id) + values (1, 0, 'ccd01'); + +insert into + detNormalizedStatImfile (det_id, iteration, class_id) + values (1, 0, 'ccd01'); + +insert into + detNormalizedStatImfile (det_id, iteration, class_id) + values (1, 0, 'ccd00'); + +update detRun set iteration = 1 where det_id = 1; + +insert into + detResidImfile (det_id, iteration, class_id) + values (1, 1, 'ccd00'); + +insert into + detResidImfile (det_id, iteration, class_id) + values (1, 1, 'ccd01'); + +insert into + detNormalizedStatImfile (det_id, iteration, class_id) + values (1, 1, 'ccd00'); + +insert into + detNormalizedStatImfile (det_id, iteration, class_id) + values (1, 1, 'ccd01'); Index: trunk/ippTools/share/chiptool_change_imfile_data_state.sql =================================================================== --- trunk/ippTools/share/chiptool_change_imfile_data_state.sql (revision 23154) +++ trunk/ippTools/share/chiptool_change_imfile_data_state.sql (revision 23229) @@ -7,8 +7,6 @@ chip_id = %lld AND class_id = '%s' - -- only update if chipRun.state has the expected value - AND ( - SELECT state from chipRun where chipRun.chip_id = chipProcessedImfile.chip_id - ) = '%s' - - +-- -- only update if chipRun.state has the expected value +-- AND ( +-- SELECT state from chipRun where chipRun.chip_id = chipProcessedImfile.chip_id +-- ) = '%s' Index: trunk/ippTools/share/dettool_processedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_processedimfile.sql (revision 23154) +++ trunk/ippTools/share/dettool_processedimfile.sql (revision 23229) @@ -1,2 +1,3 @@ +-- is this DISTINCT needed? SELECT DISTINCT detRun.det_type, Index: trunk/ippTools/share/dettool_residimfile.sql =================================================================== --- trunk/ippTools/share/dettool_residimfile.sql (revision 23154) +++ trunk/ippTools/share/dettool_residimfile.sql (revision 23229) @@ -2,9 +2,13 @@ detRun.det_type, detRun.mode, - detResidImfile.*, - rawExp.exp_time - FROM detResidImfile - JOIN detRun - USING(det_id, iteration) - JOIN rawExp - USING(exp_id) + rawExp.exp_time, + detResidImfile.* +FROM detResidImfile +JOIN detRun + USING(det_id, iteration) +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + AND detResidImfile.exp_id = detInputExp.exp_id +JOIN rawExp + ON rawExp.exp_id = detResidImfile.exp_id Index: trunk/ippTools/share/dettool_tonormalizedstat.sql =================================================================== --- trunk/ippTools/share/dettool_tonormalizedstat.sql (revision 23154) +++ trunk/ippTools/share/dettool_tonormalizedstat.sql (revision 23229) @@ -1,58 +1,33 @@ --- select detRun.det_id (det_id) --- select detRun.iteration --- by: --- find the current iteration bassed on det_id --- find all exp_ids in the current det_id/iteration from detInputExp --- sort to detInputExp.imfiles to find the largest value per det_id/iter --- compare imfiles to the number of detStackedImfiles by class_id --- and: --- ??? --- det_id is not in detStackedImfile; --- iteration is not in detStackedImfile; - -SELECT - det_id, - det_type, - iteration, - camera, - workdir, - class_id -FROM - (SELECT DISTINCT - detRun.det_id, - detRun.det_type, - detRun.iteration, - detRun.workdir, - rawExp.camera, - detStackedImfile.class_id, - rawImfile.class_id as rawimfile_class_id - FROM detRun - JOIN detInputExp - ON detRun.det_id = detInputExp.det_id - AND detRun.iteration = detInputExp.iteration - JOIN rawExp - ON detInputExp.exp_id = rawExp.exp_id - JOIN rawImfile - ON rawExp.exp_id = rawImfile.exp_id - LEFT JOIN detStackedImfile - ON detInputExp.det_id = detStackedImfile.det_id - AND detInputExp.iteration = detStackedImfile.iteration - AND rawImfile.class_id = detStackedImfile.class_id - LEFT JOIN detNormalizedStatImfile - ON detStackedImfile.det_id = detNormalizedStatImfile.det_id - AND detStackedImfile.iteration = detNormalizedStatImfile.iteration - AND detStackedImfile.class_id = detNormalizedStatImfile.class_id - WHERE - detRun.state = 'run' - AND detRun.mode = 'master' - AND detNormalizedStatImfile.det_id IS NULL - AND detNormalizedStatImfile.iteration IS NULL - AND detNormalizedStatImfile.class_id IS NULL - GROUP BY --- rawExp.exp_id, - detRun.iteration, - detRun.det_id - HAVING - COUNT(rawImfile.class_id) = COUNT(detStackedImfile.class_id) - AND SUM(detStackedImfile.fault) = 0 - ) as tonormalizedstat +-- a det_run + iteration is ready for normstat when: all detResidImfile entries for that iteration corresponding to all detProcessedImfile entries are available +SELECT + detRun.det_id, + detRun.det_type, + detRun.iteration, + detRun.camera, + detRun.workdir, + detProcessedImfile.class_id, + COUNT(detProcessedImfile.class_id), + COUNT(detResidImfile.class_id) +FROM detRun +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detProcessedImfile.det_id + AND detResidImfile.class_id = detProcessedImfile.class_id + AND detResidImfile.iteration = detRun.iteration +LEFT JOIN detNormalizedStatImfile + ON detNormalizedStatImfile.det_id = detRun.det_id + AND detNormalizedStatImfile.iteration = detRun.iteration + AND detNormalizedStatImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detNormalizedStatImfile.det_id IS NULL + AND detNormalizedStatImfile.iteration IS NULL + AND detNormalizedStatImfile.class_id IS NULL +GROUP BY + detRun.iteration, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault) = 0 Index: trunk/ippTools/share/dettool_toresidexp.sql =================================================================== --- trunk/ippTools/share/dettool_toresidexp.sql (revision 23154) +++ trunk/ippTools/share/dettool_toresidexp.sql (revision 23229) @@ -5,4 +5,6 @@ -- id, detrend type, and whether the exposure was included in the stack for -- this iteration. + +-- require the corresponding detNormalizedExp to complete before starting -- select detRun.det_id @@ -41,4 +43,6 @@ detResidImfile.class_id FROM detRun + JOIN detNormalizedExp + USING(det_id, iteration) JOIN detInputExp USING(det_id, iteration) Index: trunk/ippTools/share/dettool_toresidimfile.sql =================================================================== --- trunk/ippTools/share/dettool_toresidimfile.sql (revision 23154) +++ trunk/ippTools/share/dettool_toresidimfile.sql (revision 23229) @@ -13,5 +13,6 @@ detProcessedImfile.class_id, detProcessedImfile.uri, - detNormalizedImfile.uri AS det_uri, +-- detNormalizedImfile.uri AS det_uri, + detStackedImfile.uri AS det_uri, detRun.det_id AS ref_det_id, detRun.iteration AS ref_iter, @@ -26,11 +27,17 @@ ON detRun.det_id = detProcessedImfile.det_id AND detInputExp.exp_id = detProcessedImfile.exp_id -JOIN detNormalizedImfile - ON detRun.det_id = detNormalizedImfile.det_id - AND detRun.iteration = detNormalizedImfile.iteration - AND detProcessedImfile.class_id = detNormalizedImfile.class_id -JOIN detNormalizedExp - ON detRun.det_id = detNormalizedExp.det_id - AND detRun.iteration = detNormalizedExp.iteration +JOIN detStackedImfile + ON detRun.det_id = detStackedImfile.det_id + AND detRun.iteration = detStackedImfile.iteration + AND detProcessedImfile.class_id = detStackedImfile.class_id +-- EAM : replacing detNormalizedImfile with detStackedImfile to change the sequencing +-- JOIN detNormalizedImfile +-- ON detRun.det_id = detNormalizedImfile.det_id +-- AND detRun.iteration = detNormalizedImfile.iteration +-- AND detProcessedImfile.class_id = detNormalizedImfile.class_id +-- EAM : we there is no reason to wait for all stacks to complete before continuing +-- JOIN detNormalizedExp +-- ON detRun.det_id = detNormalizedExp.det_id +-- AND detRun.iteration = detNormalizedExp.iteration LEFT JOIN detResidImfile ON detRun.det_id = detResidImfile.det_id @@ -41,6 +48,7 @@ detRun.state = 'run' AND detRun.mode = 'master' - AND detNormalizedImfile.fault = 0 - AND detNormalizedExp.fault = 0 + AND detStackedImfile.fault = 0 +-- AND detNormalizedImfile.fault = 0 +-- AND detNormalizedExp.fault = 0 AND detResidImfile.det_id IS NULL AND detResidImfile.iteration IS NULL Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 23154) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 23229) @@ -737,7 +737,5 @@ REFERENCES detInputExp(det_id, iteration, exp_id), FOREIGN KEY (det_id, exp_id, class_id) - REFERENCES detProcessedImfile(det_id, exp_id, class_id), - FOREIGN KEY (ref_det_id, ref_iter) - REFERENCES detNormalizedExp(det_id, iteration) + REFERENCES detProcessedImfile(det_id, exp_id, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: trunk/ippTools/share/warptool_change_skyfile_data_state.sql =================================================================== --- trunk/ippTools/share/warptool_change_skyfile_data_state.sql (revision 23154) +++ trunk/ippTools/share/warptool_change_skyfile_data_state.sql (revision 23229) @@ -9,7 +9,5 @@ AND skycell_id = '%s' -- only update if chipRun.state has the expected value - AND ( - SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id - ) = '%s' - - +-- AND ( +-- SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id +-- ) = '%s' Index: trunk/ippTools/src/chiptool.c =================================================================== --- trunk/ippTools/src/chiptool.c (revision 23154) +++ trunk/ippTools/src/chiptool.c (revision 23229) @@ -52,9 +52,10 @@ static bool runMode(pxConfig *config); static bool tocleanedimfileMode(pxConfig *config); -static bool tocleanedimfile_from_scrubbedMode(pxConfig *config); +// static bool tocleanedimfile_from_scrubbedMode(pxConfig *config); static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state); # define MODECASE(caseName, func) \ @@ -92,5 +93,5 @@ MODECASE(CHIPTOOL_MODE_RUN, runMode); MODECASE(CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileMode); - MODECASE(CHIPTOOL_MODE_TOCLEANEDIMFILE_FROM_SCRUBBED, tocleanedimfile_from_scrubbedMode); + // MODECASE(CHIPTOOL_MODE_TOCLEANEDIMFILE_FROM_SCRUBBED, tocleanedimfile_from_scrubbedMode); MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); @@ -651,13 +652,36 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_S16(code, config->args, "-code", true, false); - - if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, code)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (state && code) { + psError(PS_ERR_UNKNOWN, true, "only one of -set_state and -code may be supplied"); + return false; + } + + if (state) { + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + return false; + } + if (!change_imfile_data_state(config, state, "unknown")) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); return false; - } - psFree(where); - - return true; + } + return true; + } + + if (code) { + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, code)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + psFree(where); + return true; + } + + psError(PS_ERR_UNKNOWN, true, "one of -set_state or -code must be supplied"); + return false; } @@ -1195,4 +1219,9 @@ // shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile +// XXX EAM : this function was enforcing only certain transitions with the SQL. However, +// this is getting fairly messy now that we have added a few additional target and +// destination states. I'm disabling these restrictions for now; is there are better way +// to enforce the allowed state transitions? + static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) { @@ -1210,6 +1239,8 @@ } + // XXX this feature is disabled (run_state is ignored) // note only updates if chipRun.state = run_state - if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, class_id, run_state)) { + + if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, class_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1246,8 +1277,4 @@ { return change_imfile_data_state(config, "cleaned", "goto_cleaned"); -} -static bool tocleanedimfile_from_scrubbedMode(pxConfig *config) -{ - return change_imfile_data_state(config, "cleaned", "goto_scrubbed"); } static bool tofullimfileMode(pxConfig *config) Index: trunk/ippTools/src/chiptoolConfig.c =================================================================== --- trunk/ippTools/src/chiptoolConfig.c (revision 23154) +++ trunk/ippTools/src/chiptoolConfig.c (revision 23229) @@ -172,7 +172,8 @@ // -updateprocessedimfile psMetadata *updateprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code (required)", 0); + psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -promoteexp @@ -234,9 +235,9 @@ psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); - // -tocleanedimfile_from_scrubbed - psMetadata *tocleanedimfile_from_scrubbedArgs = psMetadataAlloc(); - psMetadataAddS64(tocleanedimfile_from_scrubbedArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); - psMetadataAddStr(tocleanedimfile_from_scrubbedArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); - +// // -tocleanedimfile_from_scrubbed +// psMetadata *tocleanedimfile_from_scrubbedArgs = psMetadataAlloc(); +// psMetadataAddS64(tocleanedimfile_from_scrubbedArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); +// psMetadataAddStr(tocleanedimfile_from_scrubbedArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); +// // -tofullimfile psMetadata *tofullimfileArgs = psMetadataAlloc(); @@ -280,5 +281,5 @@ PXOPT_ADD_MODE("-run", "show runs", CHIPTOOL_MODE_RUN, runArgs); PXOPT_ADD_MODE("-tocleanedimfile", "set imfile state to cleaned", CHIPTOOL_MODE_TOCLEANEDIMFILE, tocleanedimfileArgs); - PXOPT_ADD_MODE("-tocleanedimfile_from_scrubbed", "set imfile state to cleaned (for goto_scrubbed)", CHIPTOOL_MODE_TOCLEANEDIMFILE_FROM_SCRUBBED, tocleanedimfile_from_scrubbedArgs); + // PXOPT_ADD_MODE("-tocleanedimfile_from_scrubbed", "set imfile state to cleaned (for goto_scrubbed)", CHIPTOOL_MODE_TOCLEANEDIMFILE_FROM_SCRUBBED, tocleanedimfile_from_scrubbedArgs); PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); Index: trunk/ippTools/src/dettoolConfig.c =================================================================== --- trunk/ippTools/src/dettoolConfig.c (revision 23154) +++ trunk/ippTools/src/dettoolConfig.c (revision 23229) @@ -640,13 +640,14 @@ // -residimfile psMetadata *residimfileArgs = psMetadataAlloc(); - psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); - psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); - psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); - psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-included", 0, "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); // -revertresidimfile Index: trunk/ippTools/src/dettool_processedimfile.c =================================================================== --- trunk/ippTools/src/dettool_processedimfile.c (revision 23154) +++ trunk/ippTools/src/dettool_processedimfile.c (revision 23229) @@ -165,11 +165,4 @@ PS_ASSERT_PTR_NON_NULL(config, false); - bool hasWhere = false; - - PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "=="); @@ -179,4 +172,10 @@ PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); + psString query = pxDataGet("dettool_processedimfile.sql"); if (!query) { @@ -186,4 +185,5 @@ } + bool hasWhere = false; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereSQL(where, NULL); @@ -211,4 +211,5 @@ if (faulted) { + // list only faulted rows psStringAppend(&query, " %s", " detProcessedImfile.fault != 0"); } else { Index: trunk/ippTools/src/dettool_residimfile.c =================================================================== --- trunk/ippTools/src/dettool_residimfile.c (revision 23154) +++ trunk/ippTools/src/dettool_residimfile.c (revision 23229) @@ -159,5 +159,7 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); psString query = pxDataGet("dettool_residimfile.sql"); @@ -175,4 +177,14 @@ } psFree(where); + + // restrict search to included imfiles + if (included) { + if (hasWhere) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } else { + psStringAppend(&query, " WHERE detInputExp.include = 1"); + } + hasWhere = true; + } if (hasWhere) { Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 23154) +++ trunk/ippTools/src/warptool.c (revision 23229) @@ -55,4 +55,5 @@ static bool updateskyfileMode(pxConfig *config); +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); static bool isValidMode(pxConfig *config, const char *mode); @@ -1604,4 +1605,9 @@ // shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile +// XXX EAM : this function was enforcing only certain transitions with the SQL. However, +// this is getting fairly messy now that we have added a few additional target and +// destination states. I'm disabling these restrictions for now; is there are better way +// to enforce the allowed state transitions? + static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) { @@ -1619,6 +1625,8 @@ } + // XXX this feature is disabled (run_state is ignored) // note only updates if warpRun.state = run_state - if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id, run_state)) { + + if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); // rollback @@ -1668,14 +1676,37 @@ PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, code, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - - return true; -} + if (state && code) { + psError(PS_ERR_UNKNOWN, true, "only one of -set_state and -code may be supplied"); + return false; + } + + if (state) { + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state); + return false; + } + if (!change_skyfile_data_state(config, state, "unknown")) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + return true; + } + + if (code) { + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQueryF(config->dbh, query, code, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + return true; + } + + psError(PS_ERR_UNKNOWN, true, "one of -set_state or -code must be supplied"); + return false; +} Index: trunk/ippTools/src/warptoolConfig.c =================================================================== --- trunk/ippTools/src/warptoolConfig.c (revision 23154) +++ trunk/ippTools/src/warptoolConfig.c (revision 23229) @@ -348,9 +348,10 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); - // -toupdateskyfile + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); - psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code", 0, "new fault code", 0); + psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psFree(now);