Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 29553) +++ trunk/ippTools/share/Makefile.am (revision 29561) @@ -238,4 +238,6 @@ magictool_exposure.sql \ magicdstool_clearstatefaults.sql \ + magicdstool_change_file_data_state.sql \ + magicdstool_change_run_state.sql \ magicdstool_completed_runs.sql \ magicdstool_completedrevert.sql \ @@ -252,4 +254,6 @@ magicdstool_getskycells.sql \ magicdstool_revertdestreakedfile.sql \ + magicdstool_revertupdated.sql \ + magicdstool_setfiletoupdate.sql \ magicdstool_tocleanup.sql \ magicdstool_todestreak_camera.sql \ Index: trunk/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- trunk/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29553) +++ trunk/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29561) @@ -11,6 +11,7 @@ AND (chipRun.state = 'cleaned' OR chipRun.state = 'update') AND (chipProcessedImfile.data_state = 'cleaned') - -- don't queue update if the associated magicDSFile exists + -- don't queue update if the associated magicDSFile exists and isn't cleaned AND (chipRun.magicked = 0 - OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new') - AND magicDSFile.component IS NULL) + OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + AND (magicDSFile.data_state = 'cleaned`' OR magicDSFile.data_state = 'update')) + ) Index: trunk/ippTools/share/magicdstool_change_file_data_state.sql =================================================================== --- trunk/ippTools/share/magicdstool_change_file_data_state.sql (revision 29561) +++ trunk/ippTools/share/magicdstool_change_file_data_state.sql (revision 29561) @@ -0,0 +1,8 @@ +-- handle changes in data_state. +UPDATE magicDSFile + JOIN magicDSRun USING(magic_ds_id) +SET + magicDSFile.data_state = '%s' +WHERE + magic_ds_id = %lld + AND component = '%s' Index: trunk/ippTools/share/magicdstool_change_run_state.sql =================================================================== --- trunk/ippTools/share/magicdstool_change_run_state.sql (revision 29561) +++ trunk/ippTools/share/magicdstool_change_run_state.sql (revision 29561) @@ -0,0 +1,15 @@ +-- change state of magicDSRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the constituant imfiles are in the right state +-- arguments are new state (cleaned or purged) chip_id and new state again for +-- the chipProcessedImfile sub query +UPDATE magicDSRun + SET magicDSRun.state = '%s' +WHERE + magicDSRun.magic_ds_id = %lld + AND (SELECT + COUNT(magic_ds_id) + FROM magicDSFile + WHERE + magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND data_state != '%s' + ) = 0 Index: trunk/ippTools/share/magicdstool_revertupdated.sql =================================================================== --- trunk/ippTools/share/magicdstool_revertupdated.sql (revision 29561) +++ trunk/ippTools/share/magicdstool_revertupdated.sql (revision 29561) @@ -0,0 +1,7 @@ +UPDATE magicDSRun + JOIN magicDSFile using(magic_ds_id) +SET magicDSFile.fault = 0 +WHERE + magicDSRun.state = 'update' + AND magicDSFile.data_state = 'update' + AND magicDSFile.fault != 0 Index: trunk/ippTools/share/magicdstool_setfiletoupdate.sql =================================================================== --- trunk/ippTools/share/magicdstool_setfiletoupdate.sql (revision 29561) +++ trunk/ippTools/share/magicdstool_setfiletoupdate.sql (revision 29561) @@ -0,0 +1,9 @@ +UPDATE magicDSRun + JOIN magicDSFile USING(magic_ds_id) +SET magicDSRun.state = 'update', + magicDSFile.data_state = 'update', + magicDSFile.fault = 0 + -- set hook %s +WHERE magic_ds_id = %lld + AND (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + AND (magicDSFile.data_state = 'cleaned') Index: trunk/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicDSRun.magic_id, + magicDSRun.state, chipRun.exp_id, camRun.magicked, Index: trunk/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicDSRun.magic_id, + magicDSRun.state, chipRun.exp_id, chipRun.magicked, @@ -37,9 +38,10 @@ ON magicDSRun.label = Label.label WHERE - magicDSRun.state = 'new' + ((magicDSRun.state = 'new' AND magicDSFile.component IS NULL) + OR (magicDSRun.state = 'update' AND magicDSFile.data_state = 'update' + AND magicDSFile.fault = 0)) AND magicDSRun.stage = 'chip' AND (chipRun.state = 'full' OR (chipRun.state = 'update' and chipProcessedImfile.data_state = 'full')) AND chipProcessedImfile.fault = 0 AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL AND (Label.active OR Label.active IS NULL) Index: trunk/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 29561) @@ -3,4 +3,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, @@ -50,4 +51,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: trunk/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: trunk/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: trunk/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 29553) +++ trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 29561) @@ -22,5 +22,7 @@ JOIN rawExp ON chipRun.exp_id = rawExp.exp_id WHERE magicDSRun.stage = 'chip' - AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + AND (((magicDSRun.state = 'new' OR (magicDSRun.state = 'update')) + AND magicDSFile.fault > 0) + -- why don't we require a fault for these states? OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) Index: trunk/ippTools/src/magicdstool.c =================================================================== --- trunk/ippTools/src/magicdstool.c (revision 29553) +++ trunk/ippTools/src/magicdstool.c (revision 29561) @@ -39,4 +39,5 @@ static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); +static bool updatedestreakedfileMode(pxConfig *config); static bool clearstatefaultsMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); @@ -45,4 +46,7 @@ static bool completedrevertMode(pxConfig *config); static bool tocleanupMode(pxConfig *config); +static bool tofullfileMode(pxConfig *config); +static bool tocleanedfileMode(pxConfig *config); +static bool setfiletoupdateMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); @@ -73,4 +77,5 @@ MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE,updatedestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); @@ -79,4 +84,7 @@ MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); + MODECASE(MAGICDSTOOL_MODE_TOFULLFILE, tofullfileMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANEDFILE, tocleanedfileMode); + MODECASE(MAGICDSTOOL_MODE_SETFILETOUPDATE, setfiletoupdateMode); default: psAbort("invalid option (this should not happen)"); @@ -558,4 +566,9 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + if (!strcmp(state, "update")) { + fprintf(stderr, "'-updaterun -set_state update' is not supported. Use -setfiletoupdate"); + return false; + } + // optional PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); @@ -907,4 +920,71 @@ return false; } + + return true; +} +static bool updatedestreakedfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required values + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false); + PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); + PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false); + PXOPT_LOOKUP_F32(nondiff_frac, config->args, "-nondiff_frac", false, false); + PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false); + + psString query = psStringCopy("UPDATE magicDSFile"); + + + char *initial_separator = " SET"; + char *sep = initial_separator; + if (fault) { + psStringAppend(&query, "%s fault = %d", sep, fault); + sep = ", "; + } + if (backup_path_base) { + psStringAppend(&query, "%s backup_path_base = '%s'", sep, backup_path_base); + sep = ", "; + } + if (recovery_path_base) { + psStringAppend(&query, "%s recovery_path_base = '%s'", sep, recovery_path_base); + sep = ", "; + } + if (data_state) { + psStringAppend(&query, "%s data_state = '%s'", sep, data_state); + sep = ", "; + } + if (streak_frac) { + psStringAppend(&query, "%s streak_frac = '%f'", sep, streak_frac); + sep = ", "; + } + if (nondiff_frac) { + psStringAppend(&query, "%s nondiff_frac = '%f'", sep, nondiff_frac); + sep = ", "; + } + if (run_time) { + psStringAppend(&query, "%s run_time = '%f'", sep, run_time); + sep = ", "; + } + if (sep == initial_separator) { + psFree(query); + psError(PS_ERR_UNKNOWN, true, "must set at least one value"); + return false; + } + + psStringAppend(&query, "\nWHERE magic_ds_id = %" PRId64 " AND component = '%s'\n", magic_ds_id, component); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); return true; @@ -1019,8 +1099,28 @@ PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + psString queryFile = NULL; + if (state) { + if (! strcmp(state, "new") ) { + queryFile = "magicdstool_revertdestreakedfile.sql"; + } else if (!strcmp(state, "update")) { + queryFile = "magicdstool_revertupdated.sql"; + } else { + psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state); + return false; + } + } else { + queryFile = "magicdstool_revertdestreakedfile.sql"; + } + psString query = pxDataGet(queryFile); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } if (psListLength(where->list)) { @@ -1241,4 +1341,5 @@ (strcmp(state, "censored") == 0) || (strcmp(state, "cleaned") == 0) || + (strcmp(state, "update") == 0) || (strcmp(state, "goto_restored") == 0) || (strcmp(state, "goto_censored") == 0) || @@ -1520,2 +1621,125 @@ return true; } + +// update magicDSFile.data_state to given value. +// afterwards, if all files in the magicDSRun have the new state, +// update the state for it as well +// shared code for the modes -tocleanedfile -tofullfile + +static bool change_file_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // magic_id, component + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + psString query = pxDataGet("magicdstool_change_file_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!strcmp(data_state, "full")) { + // if -tofullfile optoinally set the magicked value for the component + PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); + // set the image file's magicked flag + if (!setMagicked(config, magic_ds_id, component)) { + psError(PS_ERR_UNKNOWN, false, "setMagicked failed"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, component)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("magicdstool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedfileMode(pxConfig *config) +{ + return change_file_data_state(config, "cleaned"); +} +static bool tofullfileMode(pxConfig *config) +{ + return change_file_data_state(config, "full"); +} +/* +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedfileMode(pxConfig *config) +{ + return change_file_data_state(config, "scrubbed", "goto_scrubbed"); +} +*/ + +// a very specfic function to queue a cleaned magicDSFile to be updated +static bool setfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("magicdstool_setfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label); + } + + if (component) { + psStringAppend(&query, " AND (magicDSFile.component = '%s')", component); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} + + Index: trunk/ippTools/src/magicdstool.h =================================================================== --- trunk/ippTools/src/magicdstool.h (revision 29553) +++ trunk/ippTools/src/magicdstool.h (revision 29561) @@ -38,4 +38,8 @@ MAGICDSTOOL_MODE_COMPLETEDREVERT, MAGICDSTOOL_MODE_TOCLEANUP, + MAGICDSTOOL_MODE_TOCLEANEDFILE, + MAGICDSTOOL_MODE_TOFULLFILE, + MAGICDSTOOL_MODE_SETFILETOUPDATE, + MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE, } MAGICDStoolMode; Index: trunk/ippTools/src/magicdstoolConfig.c =================================================================== --- trunk/ippTools/src/magicdstoolConfig.c (revision 29553) +++ trunk/ippTools/src/magicdstoolConfig.c (revision 29561) @@ -137,7 +137,22 @@ psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + // -updatedestreakedfile + psMetadata *updatedestreakedfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatedestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "define component name (required)", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-backup_path_base", 0, "define backup URI", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-data_state", 0, "change data_state", NULL); + psMetadataAddBool(updatedestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-nondiff_frac", 0, "set fraction of pixels masked because nondiffed", 0); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0); + psMetadataAddS16(updatedestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + // -revertdestreakedfile psMetadata *revertdestreakedfileArgs = psMetadataAlloc(); psMetadataAddS64(revertdestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); + psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -197,4 +212,22 @@ psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tocleanedfile + psMetadata *tocleanedfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(tocleanedfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + + // -tofullfile + psMetadata *tofullfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(tofullfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddBool(tofullfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + + // -setfiletoupdate + psMetadata *setfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setfiletoupdateArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL); + psFree(now); @@ -230,4 +263,13 @@ MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); + PXOPT_ADD_MODE("-tofullfile", "set component's data_state to full", + MAGICDSTOOL_MODE_TOFULLFILE, tofullfileArgs); + PXOPT_ADD_MODE("-tocleanedfile", "set component's data_state to cleaned", + MAGICDSTOOL_MODE_TOCLEANEDFILE, tocleanedfileArgs); + PXOPT_ADD_MODE("-setfiletoupdate", "set component's data_state to update", + MAGICDSTOOL_MODE_SETFILETOUPDATE, setfiletoupdateArgs); + PXOPT_ADD_MODE("-updatedestreakedfile", "update parameters of destreaked file", + MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE, updatedestreakedfileArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {