Index: /branches/bills_branches/bills_201312/dbconfig/changes.txt =================================================================== --- /branches/bills_branches/bills_201312/dbconfig/changes.txt (revision 36416) +++ /branches/bills_branches/bills_201312/dbconfig/changes.txt (revision 36417) @@ -2436,5 +2436,4 @@ CREATE TABLE fullForceRun ( ff_id BIGINT NOT NULL AUTO_INCREMENT, - warp_id BIGINT, skycal_id BIGINT, sources_path_base VARCHAR(255), @@ -2451,11 +2450,33 @@ KEY(label), KEY(data_group), - KEY(warp_id), KEY(skycal_id), - FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( ff_id BIGINT, path_base VARCHAR(255) NOT NULL, @@ -2463,5 +2484,5 @@ quality SMALLINT NOT NULL, hostname VARCHAR(64) NOT NULL, - software_ver VARCHAR(64), + software_ver VARCHAR(16), fault SMALLINT NOT NULL, PRIMARY KEY(ff_id), Index: /branches/bills_branches/bills_201312/ippTasks/fullforce.pro =================================================================== --- /branches/bills_branches/bills_201312/ippTasks/fullforce.pro (revision 36416) +++ /branches/bills_branches/bills_201312/ippTasks/fullforce.pro (revision 36417) @@ -8,25 +8,42 @@ ### Initialise the books containing the tasks to do book init fullForceRun +book init fullForceSummary ### Database lists $fullForce_DB = 0 $fullForce_revert_DB = 0 +$fullForceSummary_DB = 0 +$fullForceSummary_revert_DB = 0 ### Check status of fullForce tasks -macro fullForce.status +macro fullforce.status book listbook fullForceRun end ### Reset fullForce tasks -macro fullForce.reset +macro fullforce.reset book init fullForceRun end +macro fullforce.summary.status + book listbook fullForceSummary +end + +### Reset fullForce tasks +macro fullforce.summary.reset + book init fullForceSummary +end ### Turn fullForce tasks on -macro fullForce.on - task fullForce.load - active true - end - task fullForce.run +macro fullforce.on + task fullforce.load + active true + end + task fullforce.run + active true + end + task fullforce.summary.load + active true + end + task fullforce.summary.run active true end @@ -34,28 +51,40 @@ ### Turn fullForce tasks off -macro fullForce.off - task fullForce.load - active false - end - task fullForce.run - active false - end -end - -macro fullForce.revert.on - task fullForce.revert - active true - end -end - -macro fullForce.revert.off - task fullForce.revert - active false - end -end - -### Load tasks for fullForce +macro fullforce.off + task fullforce.load + active false + end + task fullforce.run + active false + end + task fullforce.summary.load + active false + end + task fullforce.summary.run + active false + end +end + +macro fullforce.revert.on + task fullforce.revert + active true + end + task fullforce.summary.revert + active true + end +end + +macro fullforce.revert.off + task fullforce.revert + active false + end + task fullforce.summary.revert + active false + end +end + +### Load tasks for fullforce ### Tasks are loaded into fullForceRun. -task fullForce.load +task fullforce.load host local @@ -66,5 +95,5 @@ stdout NULL - stderr $LOGDIR/fullForce.log + stderr $LOGDIR/fullforce.log task.exec @@ -88,5 +117,5 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout fullForceRun -key ff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout fullForceRun -key ff_id:warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) book listbook fullForceRun @@ -112,7 +141,7 @@ end -### Run tasks for the fullForce analysis +### Run tasks for the fullforce analysis ### Tasks are taken from fullForceRun. -task fullForce.run +task fullforce.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -152,8 +181,8 @@ sprintf outroot "%s/%s/%s/%s.%s.wrp.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $WARP_ID $FF_ID - stdout $LOGDIR/fullForce.log - stderr $LOGDIR/fullForce.log - - $run = psphot_fullforce_warp.pl --ff_id $FF_ID --outroot $outroot --redirect-output --camera $CAMERA --warp_path_base $WARP_PATH_BASE --sources_path_base $SOURCES_PATH_BASE --skycell_id $SKYCELL_ID --threads @MAX_THREADS@ + stdout $LOGDIR/fullforce.log + stderr $LOGDIR/fullforce.log + + $run = psphot_fullforce_warp.pl --ff_id $FF_ID --warp_id $WARP_ID --outroot $outroot --redirect-output --camera $CAMERA --warp_path_base $WARP_PATH_BASE --sources_path_base $SOURCES_PATH_BASE --skycell_id $SKYCELL_ID --threads @MAX_THREADS@ if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION @@ -192,5 +221,5 @@ end -task fullForce.revert +task fullforce.revert host local @@ -202,5 +231,5 @@ stdout NULL - stderr $LOGDIR/revert.log + stderr $LOGDIR/fullforce.revert.log task.exec @@ -243,2 +272,187 @@ end +### Load tasks for fullForceSummary +### Tasks are loaded into fullForceSummary. +task fullforce.summary.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/fullforce.summary.log + + task.exec + if ($LABEL:n == 0) break + $run = fftool -toadvance + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_DB + $run = $run -dbname $DB:$fullForceSummary_DB + $fullForceSummary_DB ++ + if ($fullForceSummary_DB >= $DB:n) set fullForceSummary_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout fullForceSummary -key ff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook fullForceSummary + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup fullForceSummary + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the fullForceSummary analyasis +### Tasks are taken from fullForceSummary. +task fullforce.summary.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + # if we are unable to run use "long" exectime + periods -exec $RUNEXEC + book npages fullForceSummary -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + + # look for new entries in fullForceSummary (pantaskState == INIT) + book getpage fullForceSummary 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fullForceSummary $pageName pantaskState RUN + book getword fullForceSummary $pageName ff_id -var FF_ID + book getword fullForceSummary $pageName tess_id -var TESS_DIR + book getword fullForceSummary $pageName skycell_id -var SKYCELL_ID + book getword fullForceSummary $pageName camera -var CAMERA + book getword fullForceSummary $pageName workdir -var WORKDIR_TEMPLATE + book getword fullForceSummary $pageName reduction -var REDUCTION + book getword fullForceSummary $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash + # set.host.for.skycell $SKYCELL_ID + host anyhost + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.ff.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $FF_ID + + stdout $LOGDIR/fullforce.summary.log + stderr $LOGDIR/fullforce.summary.log + + $run = psphot_fullforce_summary.pl --ff_id $FF_ID --outroot $outroot --redirect-output --camera $CAMERA --threads @MAX_THREADS@ + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # since we have work to do shorten exec time + periods -exec .05 + command $run + end + + # default exit status + task.exit default + process_exit fullForceSummary $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword fullForceSummary $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword fullForceSummary $options:0 pantaskState TIMEOUT + end +end + +task fullforce.summary.revert + host local + + periods -poll 10.0 + periods -exec 1200.0 + periods -timeout 120.0 + npending 1 + active true + + stdout NULL + stderr $LOGDIR/fullforce.summary.revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = fftool -revertsummary -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$fullForceSummary_revert_DB + $run = $run -dbname $DB:$fullForceSummary_revert_DB + $fullForceSummary_revert_DB ++ + if ($fullForceSummary_revert_DB >= $DB:n) set fullForceSummary_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/bills_branches/bills_201312/ippTools/share/Makefile.am =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/Makefile.am (revision 36416) +++ /branches/bills_branches/bills_201312/ippTools/share/Makefile.am (revision 36417) @@ -493,4 +493,7 @@ fftool_result.sql \ fftool_revert.sql \ - fftool_todo.sql + fftool_revertsummary.sql \ + fftool_summary.sql \ + fftool_todo.sql \ + fftool_toadvance.sql Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_revertsummary.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_revertsummary.sql (revision 36417) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_revertsummary.sql (revision 36417) @@ -0,0 +1,7 @@ +DELETE FROM fullForceSummary +USING fullForceSummary, fullForceRun, skycalRun, stackRun, skycell +WHERE fullForceRun.ff_id = fullForceSummary.ff_id + AND fullForceSummary.fault != 0 + AND fullForceRun.skycal_id = skycalRun.skycal_id + AND skycalRun.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id AND stackRun.skycell_id = skycell.skycell_id Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_summary.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_summary.sql (revision 36417) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_summary.sql (revision 36417) @@ -0,0 +1,21 @@ +SELECT DISTINCT + fullForceSummary.*, + fullForceRun.state, + fullForceRun.label, + fullForceRun.data_group, + fullForceRun.skycal_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceSummary USING(ff_id) + JOIN fullForceInput USING(ff_id) + JOIN skycalRun using(skycal_id) + JOIN stackRun using(stack_id) + JOIN warpRun using(warp_id, tess_id) + JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) + JOIN fakeRun using(fake_id) + JOIN camRun using(cam_id) + JOIN chipRun using(chip_id) + JOIN rawExp using(exp_id) Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_summaryinputs.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_summaryinputs.sql (revision 36417) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_summaryinputs.sql (revision 36417) @@ -0,0 +1,4 @@ +SELECT + fullForceResult.* +FROM fullForceRun + JOIN fullForceResult USING(ff_id) Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_toadvance.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_toadvance.sql (revision 36417) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_toadvance.sql (revision 36417) @@ -0,0 +1,21 @@ +SELECT + fullForceRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + rawExp.camera +FROM fullForceRun + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + LEFT JOIN fullForceResult USING(ff_id, warp_id) +WHERE fullForceRun.state = 'new' + -- WHERE hook %s + GROUP BY ff_id + HAVING COUNT(fullForceInput.warp_id) = COUNT(fullForceResult.warp_id) + AND SUM(fullForceResult.fault) = 0 Index: /branches/bills_branches/bills_201312/ippTools/src/fftool.c =================================================================== --- /branches/bills_branches/bills_201312/ippTools/src/fftool.c (revision 36416) +++ /branches/bills_branches/bills_201312/ippTools/src/fftool.c (revision 36417) @@ -39,6 +39,11 @@ static bool revertMode(pxConfig *config); static bool updateresultMode(pxConfig *config); - -// static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); +static bool toadvanceMode(pxConfig *config); +static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); +static bool updatesummaryMode(pxConfig *config); +static bool summaryMode(pxConfig *config); + +static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); # define MODECASE(caseName, func) \ @@ -67,4 +72,9 @@ MODECASE(FFTOOL_MODE_REVERT, revertMode); MODECASE(FFTOOL_MODE_UPDATERESULT, updateresultMode); + MODECASE(FFTOOL_MODE_TOADVANCE, toadvanceMode); + MODECASE(FFTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(FFTOOL_MODE_REVERTSUMMARY, revertsummaryMode); + MODECASE(FFTOOL_MODE_UPDATESUMMARY, updatesummaryMode); + MODECASE(FFTOOL_MODE_SUMMARY, summaryMode); default: psAbort("invalid option (this should not happen)"); @@ -403,5 +413,4 @@ } - // the where clause is required and matches the WHERE hook format string if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -462,9 +471,4 @@ // XXX not sure we need a transaction here... - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!fullForceResultInsert(config->dbh, ff_id, @@ -484,22 +488,4 @@ } -#ifdef notdef - if (!fault) { - if (!setfullForceRunState(config, ff_id, "full")) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to change staticskyRun state"); - return false; - } - } -#endif - - // point of no return - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - return true; } @@ -594,4 +580,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceResult.warp_id", "=="); pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fullForceResult.fault", "=="); @@ -645,6 +632,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-warp_id", "ff_id", "=="); - + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); if (!pxSetFaultCode(config->dbh, "fullForceResult", where, fault, quality)) { @@ -656,5 +642,5 @@ return true; } -#ifdef notyet + static bool setfullForceRunState(pxConfig *config, psS64 ff_id, const char *state) { @@ -667,4 +653,282 @@ return true; } -#endif - + +static bool toadvanceMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-ff_id", "ff_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "fullForceRun.label", "=="); + pxskycellAddWhere(config, whereMD); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_toadvance.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = NULL; + psString temp = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&whereClause, "\n AND %s", temp); + psFree(temp); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + // the where clause is required and is added to the query by the "WHERE hook" format string + if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "fullForceRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + // required + PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + + // optional + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", true, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(software_ver, config->args, "-software_ver", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fullForceSummaryInsert(config->dbh, + ff_id, + path_base, + dtime_script, + quality, + hostname, + software_ver, + fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!fault) { + if (!setfullForceRunState(config, ff_id, "full")) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to change staticskyRun state"); + return false; + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fullForceSummary.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("fftool_revertsummary.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("fftool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} +static bool updatesummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + + + if (!pxSetFaultCode(config->dbh, "fullForceSummary", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool summaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "fullForceInput.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "fullForceRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "fullForceRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "fullForceRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + pxskycellAddWhere(config, where); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("fftool_summary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("fftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "fullForceSummary", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/bills_branches/bills_201312/ippTools/src/fftool.h =================================================================== --- /branches/bills_branches/bills_201312/ippTools/src/fftool.h (revision 36416) +++ /branches/bills_branches/bills_201312/ippTools/src/fftool.h (revision 36417) @@ -32,4 +32,9 @@ FFTOOL_MODE_REVERT, FFTOOL_MODE_UPDATERESULT, + FFTOOL_MODE_TOADVANCE, + FFTOOL_MODE_ADDSUMMARY, + FFTOOL_MODE_REVERTSUMMARY, + FFTOOL_MODE_UPDATESUMMARY, + FFTOOL_MODE_SUMMARY } fftoolMode; Index: /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c =================================================================== --- /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c (revision 36416) +++ /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c (revision 36417) @@ -123,4 +123,5 @@ psMetadata *revertArgs= psMetadataAlloc(); psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -134,4 +135,55 @@ psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality code", 0); + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(toadvanceArgs); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addsummary + psMetadata *addsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); + psMetadataAddF32(addsummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(addsummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -updatesummary + psMetadata *updatesummaryArgs = psMetadataAlloc(); + psMetadataAddS64(updatesummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddF32(updatesummaryArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddStr(updatesummaryArgs, PS_LIST_TAIL, "-software_ver", 0, "define software version", 0); + psMetadataAddS16(updatesummaryArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -summary + psMetadata *summaryArgs= psMetadataAlloc(); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(summaryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(summaryArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + pxskycellAddArguments(summaryArgs); + psMetadataAddS16(summaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(summaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(summaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -revertsummary + psMetadata *revertsummaryArgs= psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-ff_id", 0, "search by full force ID", 0); + psMetadataAddStr(revertsummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddS16(revertsummaryArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertsummaryArgs); + psFree(now); @@ -142,8 +194,13 @@ PXOPT_ADD_MODE("-updaterun", "Update a run", FFTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-todo", "Get runs to do", FFTOOL_MODE_TODO, todoArgs); - PXOPT_ADD_MODE("-addresult", "Add result of run", FFTOOL_MODE_ADDRESULT, addresultArgs); - PXOPT_ADD_MODE("-result", "Get result of run", FFTOOL_MODE_RESULT, resultArgs); - PXOPT_ADD_MODE("-revert", "Revert failed run", FFTOOL_MODE_REVERT, revertArgs); - PXOPT_ADD_MODE("-updateresult", "Update fault for run", FFTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-addresult", "Add result for fullforce run on a warp", FFTOOL_MODE_ADDRESULT, addresultArgs); + PXOPT_ADD_MODE("-result", "Get result fullforce run on a warp", FFTOOL_MODE_RESULT, resultArgs); + PXOPT_ADD_MODE("-revert", "Revert failed fullforce run on a warp", FFTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updateresult", "Update result for fullforce run on a warp", FFTOOL_MODE_UPDATERESULT, updateresultArgs); + PXOPT_ADD_MODE("-toadvance", "list completed runs to summarize", FFTOOL_MODE_TOADVANCE, toadvanceArgs); + PXOPT_ADD_MODE("-addsummary", "insert summary result", FFTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-updatesummary", "update summary result", FFTOOL_MODE_UPDATESUMMARY, updatesummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert faulted summary", FFTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-summary", "list summary results", FFTOOL_MODE_SUMMARY, summaryArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {