Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 27873) +++ /trunk/dbconfig/changes.txt (revision 27874) @@ -1615,2 +1615,3 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +ALTER TABLE pstampRequest ADD COLUMN outdir VARCHAR(64) after uri; Index: /trunk/dbconfig/pstamp.md =================================================================== --- /trunk/dbconfig/pstamp.md (revision 27873) +++ /trunk/dbconfig/pstamp.md (revision 27874) @@ -30,4 +30,5 @@ outProduct STR 64 uri STR 255 + outdir STR 255 fault S32 0 END Index: /trunk/ippTasks/pstamp.pro =================================================================== --- /trunk/ippTasks/pstamp.pro (revision 27873) +++ /trunk/ippTasks/pstamp.pro (revision 27874) @@ -14,4 +14,5 @@ $pstampRev_DB = 0 $pstampDep_DB = 0 +$pstampCleanup_DB = 0 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config @@ -30,4 +31,5 @@ book init pstampFinish book init pstampDependent + book init pstampCleanup end @@ -58,7 +60,4 @@ active true end - task pstamp.job.revert - active true - end task pstamp.dependent.load active true @@ -94,5 +93,8 @@ end task pstamp.dependent.load - active true + active false + end + task pstamp.dependent.run + active false end end @@ -105,4 +107,20 @@ macro pstamp.revert.off task pstamp.job.revert + active false + end +end +macro pstamp.cleanup.on + task pstamp.cleanup.load + active true + end + task pstamp.cleanup.run + active true + end +end +macro pstamp.cleanup.off + task pstamp.cleanup.run + active false + end + task pstamp.cleanup.run active false end @@ -157,5 +175,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.request.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.request.load.log $run = pstamptool -pendingreq @@ -200,8 +218,7 @@ periods -exec $RUNEXEC periods -timeout 300 - npending 6 - - task.exec - stdout $LOGSUBDIR/pstamp.request.run.log + + task.exec + stdout NULL stderr $LOGSUBDIR/pstamp.request.run.log book npages pstampRequest -var N @@ -216,9 +233,10 @@ book getword pstampRequest $pageName uri -var URI book getword pstampRequest $pageName ds_outProduct -var PRODUCT + book getword pstampRequest $pageName outdir -var OUTDIR book getword pstampRequest $pageName label -var LABEL host anyhost - $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --label $LABEL --redirect-output + $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output add_standard_args run @@ -261,5 +279,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.finish.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.finish.load.log $run = pstamptool -completedreq @@ -306,5 +324,5 @@ task.exec - stdout $LOGSUBDIR/request.finish.run.log + stdout NULL stderr $LOGSUBDIR/request.finish.run.log book npages pstampFinish -var N @@ -321,6 +339,7 @@ book getword pstampFinish $pageName name -var REQ_NAME book getword pstampFinish $pageName outProduct -var PRODUCT - - $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --redirect-output + book getword pstampFinish $pageName outdir -var OUTDIR + + $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR --redirect-output add_standard_args run @@ -363,5 +382,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.load.log $run = pstamptool -pendingjob @@ -411,5 +430,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.run.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.run.log book npages pstampJob -var N @@ -489,5 +508,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.job.revert.log + stdout NULL stderr $LOGSUBDIR/pstamp.job.revert.log $run = pstamptool -revertjob -all @@ -532,5 +551,5 @@ task.exec - stdout $LOGSUBDIR/pstamp.dependent.load.log + stdout NULL stderr $LOGSUBDIR/pstamp.dependent.load.log $run = pstamptool -pendingdependent @@ -660,2 +679,118 @@ end +task pstamp.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + + task.exec + stdout NULL + stderr $LOGSUBDIR/pstamp.cleanup.load.log + $run = pstamptool -pendingcleanup + if ($DB:n == 0) + option DEFAULT + else + option $DB:$pstampCleanup_DB + $run = $run $PS_DBSERVER -dbname $DB:$pstampCleanup_DB + $pstampCleanup_DB ++ + if ($pstampCleanup_DB >= $DB:n) set pstampCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + task.exit $EXIT_SUCCESS + ipptool2book stdout pstampCleanup -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + echo starting request + book listbook pstampCleanup + end + + process_cleanup pstampCleanup + end + + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + task.exit timeout + showcommand timeout + end +end + +task pstamp.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + # since everything is on one file system keep npending low to avoid + # overloading nfs + + npending 10 + + task.exec + book npages pstampCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + book getpage pstampCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pstampCleanup $pageName pantaskState RUN + book getword pstampCleanup $pageName req_id -var REQ_ID + book getword pstampCleanup $pageName dbname -var DBNAME + book getword pstampCleanup $pageName name -var NAME + book getword pstampCleanup $pageName outdir -var OUTDIR + book getword pstampCleanup $pageName uri -var URI + book getword pstampCleanup $pageName reqType -var REQTYPE + book getword pstampCleanup $pageName outProduct -var PRODUCT + + # XXX: have the script set this up this + $MYLOGFILE=/data/ippdb02.0/pstamp/work/logs/cleanup.$REQ_ID + stdout $MYLOGFILE + stderr $MYLOGFILE + + host anyhost + + $run = pstamp_cleanup.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --name $NAME --outdir $OUTDIR --reqType $REQTYPE + + add_standard_args run + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + + task.exit $EXIT_SUCCESS + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pstampCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pstampCleanup $options:0 pantaskState CRASH + end + + task.exit timeout + showcommand timeout + book setword pstampCleanup $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 27873) +++ /trunk/ippTools/share/Makefile.am (revision 27874) @@ -226,4 +226,5 @@ pstamptool_getdependent.sql \ pstamptool_listjob.sql \ + pstamptool_pendingcleanup.sql \ pstamptool_pendingdependent.sql \ pstamptool_pendingjob.sql \ Index: /trunk/ippTools/share/pstamptool_pendingcleanup.sql =================================================================== --- /trunk/ippTools/share/pstamptool_pendingcleanup.sql (revision 27874) +++ /trunk/ippTools/share/pstamptool_pendingcleanup.sql (revision 27874) @@ -0,0 +1,9 @@ +-- postage stamp requests pending cleanup +SELECT + pstampRequest.*, + IFNULL(Label.priority, 10000) AS priority +FROM pstampRequest + LEFT JOIN Label ON pstampRequest.label = Label.label +WHERE pstampRequest.state = 'goto_cleaned' + AND pstampRequest.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: /trunk/ippTools/share/pstamptool_pendingdependent.sql =================================================================== --- /trunk/ippTools/share/pstamptool_pendingdependent.sql (revision 27873) +++ /trunk/ippTools/share/pstamptool_pendingdependent.sql (revision 27874) @@ -1,3 +1,4 @@ -SELECT DISTINCT pstampDependent.*, +SELECT DISTINCT + pstampDependent.*, IFNULL(Label.priority, 10000) AS priority FROM pstampDependent Index: /trunk/ippTools/share/pstamptool_pendingreq.sql =================================================================== --- /trunk/ippTools/share/pstamptool_pendingreq.sql (revision 27873) +++ /trunk/ippTools/share/pstamptool_pendingreq.sql (revision 27874) @@ -4,5 +4,5 @@ pstampDataStore.outProduct AS ds_outProduct, IFNULL(Label.priority, 10000) AS priority - FROM pstampRequest +FROM pstampRequest LEFT JOIN pstampDataStore USING(ds_id) LEFT JOIN Label ON pstampRequest.label = Label.label Index: /trunk/ippTools/share/pstamptool_updatejob.sql =================================================================== --- /trunk/ippTools/share/pstamptool_updatejob.sql (revision 27873) +++ /trunk/ippTools/share/pstamptool_updatejob.sql (revision 27874) @@ -1,3 +1,2 @@ UPDATE pstampJob LEFT JOIN pstampDependent USING(dep_id) -SET pstampJob.state = '%s' - -- set state hook %s +SET Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 27873) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 27874) @@ -1303,4 +1303,5 @@ outProduct VARCHAR(64), uri VARCHAR(255), + outdir VARCHAR(255), fault SMALLINT, PRIMARY KEY(req_id), Index: /trunk/ippTools/src/pstamptool.c =================================================================== --- /trunk/ippTools/src/pstamptool.c (revision 27873) +++ /trunk/ippTools/src/pstamptool.c (revision 27874) @@ -40,4 +40,5 @@ static bool updatereqMode(pxConfig *config); static bool revertreqMode(pxConfig *config); +static bool pendingcleanupMode(pxConfig *config); static bool addjobMode(pxConfig *config); static bool listjobMode(pxConfig *config); @@ -81,4 +82,5 @@ MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode); MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode); MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode); MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode); @@ -265,4 +267,5 @@ NULL, // outProduct uri, + NULL, // outdir 0 // fault )) { @@ -341,5 +344,5 @@ } -static bool listreqMode(pxConfig *config) +static bool pendingcleanupMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -347,21 +350,23 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); - PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - if (!psListLength(where->list)) { - psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); - return false; - } - - psString query = psStringCopy("SELECT * from pstampRequest"); - - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - psFree(where); + psString query = pxDataGet("pstamptool_pendingcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); // treat limit == 0 as "no limit" @@ -385,10 +390,7 @@ } if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "request not found"); - // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for - psError(PXTOOLS_ERR_CONFIG, true, "request not found"); - psFree(output); - // we return false so that the caller can determine that a request does not exist - return false; + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; } @@ -405,24 +407,27 @@ } -static bool completedreqMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); +static bool listreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("pstamptool_completedreq.sql"); - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - psStringAppend(&query, " ORDER BY priority DESC, req_id"); + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + return false; + } + + psString query = psStringCopy("SELECT * from pstampRequest"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); // treat limit == 0 as "no limit" @@ -438,4 +443,5 @@ return false; } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -444,9 +450,11 @@ return false; } - if (!psArrayLength(output)) { - psTrace("pstamptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("pstamptool", PS_LOG_INFO, "request not found"); + // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + psError(PXTOOLS_ERR_CONFIG, true, "request not found"); + psFree(output); + // we return false so that the caller can determine that a request does not exist + return false; } @@ -463,23 +471,103 @@ } +static bool completedreqMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_completedreq.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, " ORDER BY priority DESC, req_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + static bool updatereqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); - PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); - PXOPT_LOOKUP_STR(reqType, config->args, "-reqType", false, false); - - psString query = NULL; - psStringAppend(&query, "UPDATE pstampRequest SET"); - + // PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-set_name", false, false); + PXOPT_LOOKUP_STR(reqType, config->args, "-set_reqType", false, false); + PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_max", "req_id", "<="); + PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE"); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE pstampRequest SET"); + + psString stateCheck = NULL; char c = ' '; if (state) { psStringAppend(&query, "%c state = '%s'", c, state); c = ','; + if (!strcmp(state, "goto_cleaned")) { + psStringAppend(&stateCheck, " AND state != 'cleaned'"); + } + } + if (label) { + psStringAppend(&query, "%c label = '%s'", c, label); + c = ','; } if (outProduct) { @@ -487,5 +575,16 @@ c = ','; } - if (fault) { + if (outdir) { + psStringAppend(&query, "%c outdir = '%s'", c, outdir); + c = ','; + } + if (clearfault) { + if (fault) { + psError(PXTOOLS_ERR_CONFIG, true, "only one of -fault and -clearfault is allowed"); + return false; + } + psStringAppend(&query, "%c fault = 0", c); + c = ','; + } else if (fault) { psStringAppend(&query, "%c fault = %d", c, fault); c = ','; @@ -503,10 +602,17 @@ c = ','; } - if (!state && !outProduct && !fault && !uri && !name && !reqType) { - psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified"); - return false; - } - - psStringAppend(&query, " WHERE req_id = %" PRId64, req_id); + if (c != ',') { + psError(PS_ERR_UNKNOWN, true, "at least one set option is required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + if (stateCheck) { + psStringAppend(&query, stateCheck); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -516,11 +622,6 @@ } - psU64 affected = psDBAffectedRows(config->dbh); - // note zero is not an error - if (affected > 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" - PRIu64 " rows were modified", affected); - return false; - } + // psU64 affected = psDBAffectedRows(config->dbh); + // psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampRequests", affected); return true; @@ -778,6 +879,12 @@ return false; } - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", false, false); + + if (!state && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of -set_state and -set_fault is required"); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -786,13 +893,16 @@ PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); - psString faultStr = NULL; - if (!fault) { - faultStr = psStringCopy(""); - } else { - psStringAppend(&faultStr, "\n, pstampJob.fault = %d", fault); - } - psString query = pxDataGet("pstamptool_updatejob.sql"); + char c = ' '; + if (state) { + psStringAppend(&query, "\n %c pstampJob.state = '%s'", c, state); + c = ','; + } + if (fault) { + psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault); + c = ','; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " WHERE %s", whereClause); @@ -800,10 +910,9 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state, faultStr)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); return false; } - psFree(faultStr); psFree(query); @@ -1159,5 +1268,5 @@ PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); if (!fault && !state) { Index: /trunk/ippTools/src/pstamptool.h =================================================================== --- /trunk/ippTools/src/pstamptool.h (revision 27873) +++ /trunk/ippTools/src/pstamptool.h (revision 27874) @@ -34,4 +34,5 @@ PSTAMPTOOL_MODE_UPDATEREQ, PSTAMPTOOL_MODE_REVERTREQ, + PSTAMPTOOL_MODE_PENDINGCLEANUP, PSTAMPTOOL_MODE_ADDJOB, PSTAMPTOOL_MODE_LISTJOB, Index: /trunk/ippTools/src/pstamptoolConfig.c =================================================================== --- /trunk/ippTools/src/pstamptoolConfig.c (revision 27873) +++ /trunk/ippTools/src/pstamptoolConfig.c (revision 27874) @@ -98,11 +98,18 @@ // -updatereq psMetadata *updatereqArgs = psMetadataAlloc(); - psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to change state", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0, "define request outProduct", NULL); - psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "define request fault code", 0); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0, "define the uri", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "define the name", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "define the reqType", NULL); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to update", 0); + psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id_max", 0, "maximum req_id for which to update", 0); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_label", 0, "new label", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outProduct", 0, "new outProduct", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outdir", 0, "new outdir", NULL); + psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault code", 0); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0, "new uri", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0, "new name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0, "new reqType", NULL); + psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -revertreq @@ -112,4 +119,12 @@ psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + + // -pendingcleanup + psMetadata *pendingcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-req_id", 0, "define req_id", 0); + psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-ds_id", 0, "define ds_id", 0); + psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL); + psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addjob @@ -145,6 +160,6 @@ psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0, "dep_id for which to change state", 0); - psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0, "new state", NULL); + psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0, "new result", 0); // -revertjob @@ -160,4 +175,6 @@ // -getdependent psMetadata *getdependentArgs = psMetadataAlloc(); + // arguabely all of these could be -set_ arguments, but since this mode is both a query and + // a create if doesn't exist I like it this way psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage id for dependent (required)", 0); psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent (required)", NULL); @@ -173,5 +190,5 @@ psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0); psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); - psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-fault", 0, "new value for fault", 0); + psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-set_fault", 0, "new value for fault", 0); // -revertdependent @@ -227,4 +244,5 @@ PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); PXOPT_ADD_MODE("-revertreq", "", PSTAMPTOOL_MODE_REVERTREQ, revertreqArgs); + PXOPT_ADD_MODE("-pendingcleanup", "", PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs); PXOPT_ADD_MODE("-addjob", "", PSTAMPTOOL_MODE_ADDJOB, addjobArgs); Index: /trunk/pstamp/scripts/Makefile.am =================================================================== --- /trunk/pstamp/scripts/Makefile.am (revision 27873) +++ /trunk/pstamp/scripts/Makefile.am (revision 27874) @@ -3,4 +3,5 @@ install_files = \ + pstamp_cleanup.pl \ pstamp_finish.pl \ pstamp_insert_request.pl \ Index: /trunk/pstamp/scripts/dquery_finish.pl =================================================================== --- /trunk/pstamp/scripts/dquery_finish.pl (revision 27873) +++ /trunk/pstamp/scripts/dquery_finish.pl (revision 27874) @@ -20,5 +20,5 @@ use PS::IPP::Config qw( :standard ); -my ( $req_id, $req_name, $req_file, $product, $out_dir, $dbname, $dbserver, $verbose, $save_temps ); +my ( $req_id, $req_name, $req_file, $product, $outdir, $dbname, $dbserver, $verbose, $save_temps ); GetOptions( @@ -27,5 +27,5 @@ 'req_file=s' => \$req_file, 'product=s' => \$product, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'dbname=s' => \$dbname, 'dbserver=s' => \$dbserver, @@ -41,5 +41,5 @@ $err .= "--req_name is required\n" if !$req_name; $err .= "--product is required\n" if !$product; -$err .= "--out_dir is required\n" if !$out_dir; +$err .= "--outdir is required\n" if !$outdir; die "$err" if $err; @@ -76,16 +76,16 @@ die "product directory does not exist $prod_dir"; } -my $out_dir = "$prod_dir/$req_name"; +my $outdir = "$prod_dir/$req_name"; } -if (! -e $out_dir ) { +if (! -e $outdir ) { # something must have gone wrong at the parse stage - print STDERR "output fileset directory $out_dir does not exist\n" if $verbose; - if (! mkdir $out_dir ) { + print STDERR "output fileset directory $outdir does not exist\n" if $verbose; + if (! mkdir $outdir ) { stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose); - die "cannot create output directory $out_dir"; + die "cannot create output directory $outdir"; } -} elsif (! -d $out_dir) { +} elsif (! -d $outdir) { stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose); - die "output fileset directory $out_dir exists but is not a directory"; + die "output fileset directory $outdir exists but is not a directory"; } @@ -118,10 +118,10 @@ # XXX: have the jobs produce the reglist as with postage stamp requests -my ($REGLIST, $reg_list) = tempfile("$out_dir/reqlist.XXXX", UNLINK => !$save_temps); +my ($REGLIST, $reg_list) = tempfile("$outdir/reqlist.XXXX", UNLINK => !$save_temps); foreach my $job (@jobs) { my $job_id = $job->{job_id}; my $response_file = "response${job_id}.fits"; - my $response_path = "$out_dir/$response_file"; + my $response_path = "$outdir/$response_file"; if (-e $response_path) { @@ -131,5 +131,5 @@ # do the same if we have an error file. Should the parse data be uploaded as well? my $err_file = "parse_error.txt"; - if (-e "$out_dir/$err_file") { + if (-e "$outdir/$err_file") { print $REGLIST "$err_file|||text|\n"; } @@ -143,5 +143,5 @@ if (-s $reg_list) { my $command = "$dsreg --add $req_name --product $product --list $reg_list"; - $command .= " --copy --datapath $out_dir"; + $command .= " --copy --datapath $outdir"; $command .= " --type MOPS_DETECTABILITY_RESPONSE"; $command .= " --ps0 $req_id"; @@ -166,6 +166,6 @@ my $verbose = shift; - my $command = "$pstamptool -updatereq -req_id $req_id -state stop"; - $command .= " -fault $fault" if $fault; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop"; + $command .= " -set_fault $fault" if $fault; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; Index: /trunk/pstamp/scripts/dqueryparse.pl =================================================================== --- /trunk/pstamp/scripts/dqueryparse.pl (revision 27873) +++ /trunk/pstamp/scripts/dqueryparse.pl (revision 27874) @@ -26,5 +26,5 @@ my ($no_update, $imagedb, $label); -my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps); +my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps); my ($job_id,$rownum); # stuff from the post-update world # @@ -37,5 +37,5 @@ 'job_id=s' => \$job_id, 'rownum=s' => \$rownum, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'label=s' => \$label, 'product=s' => \$product, @@ -52,5 +52,5 @@ if ($mode ne "list_uri") { die "req_id is required" if !$req_id; - die "out_dir is required" if !$out_dir; + die "outdir is required" if !$outdir; die "product is required" if !$product; } @@ -80,7 +80,7 @@ # Unless we're running as a job, write the parse arguments in case we need to rerun this parsing. if (!$job_id) { - my $argslist = "$out_dir/parse.args"; + my $argslist = "$outdir/parse.args"; open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); - print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --out_dir $out_dir --file $req_file\n"; + print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n"; close ARGSLIST; } @@ -105,17 +105,17 @@ # Set up the workdir for this query. -if (! -e $out_dir ) { - mkdir $out_dir or my_die("cannot create output directory $out_dir", $PS_EXIT_PROG_ERROR); -} elsif (! -d $out_dir ) { - my_die ("output fileset directory $out_dir exists but is not a directory", $PS_EXIT_PROG_ERROR); +if (! -e $outdir ) { + mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR); +} elsif (! -d $outdir ) { + my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR); } # Pass along the request file to the response generator. -my $response_file = "$out_dir/${req_name}.dresponse.${req_id}.fits"; +my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits"; my $fault; my $data_to_update = ''; { - my $command = "$detectresponse --input $req_file --output $response_file --workdir $out_dir"; + my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir"; $command .= " --save-temps" if $save_temps; $command .= " --verbose" if $verbose; @@ -137,5 +137,5 @@ # for the completed work, and move the response to a standardized name. if ($fault == 0) { - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; + my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; $command .= " -job_type detect_query -state stop -fault 0"; $command .= " -rownum 1"; @@ -147,5 +147,5 @@ chomp $job_id; if ($job_id && -e $response_file) { - rename $response_file, "$out_dir/response${job_id}.fits"; + rename $response_file, "$outdir/response${job_id}.fits"; } $result = 0; @@ -158,8 +158,8 @@ # Failed to run correctly, which means that we need to queue a job and flag data for updating. # Get the dependency id for the data we're requesting be updated. - my $dep_id = queue_update_run($req_id,$job_id,$out_dir,$label,$data_to_update); + my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update); # Link this request to a job and link that job to any dependency - my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; + my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; $command .= " -job_type detect_query -state run -fault 0"; $command .= " -rownum 1"; @@ -174,5 +174,5 @@ if ($job_id && -e $response_file) { # We shouldn't have a response file at this stage. - rename $response_file, "$out_dir/response${job_id}.fits"; + rename $response_file, "$outdir/response${job_id}.fits"; } $result = 0; @@ -186,6 +186,6 @@ # which will notice that we've inserted the stopped job and decide we're finished. Easy enough. { - my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name -outProduct $product"; - $command .= " -fault $result" if $result; + my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product"; + $command .= " -set_fault $result" if $result; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -201,5 +201,5 @@ # If we have to queue an update run, do so and create a new dependent sub queue_update_run { - my ($req_id, $job_id, $out_dir, $label, $data_to_update) = @_; + my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_; my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update; @@ -211,5 +211,5 @@ my $dep_id; my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component "; - $command .= " -outdir $out_dir"; + $command .= " -outdir $outdir"; $command .= " -need_magic" if $need_magic; Index: /trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 27874) @@ -575,5 +575,5 @@ my ($state, $stage, $stage_id, $job_fault) = @_; - my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id"; + my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -642,5 +642,5 @@ carp $msg; - my $command = "$pstamptool -updatedependent -fault $fault -dep_id $dep_id"; + my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; Index: /trunk/pstamp/scripts/pstamp_cleanup.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_cleanup.pl (revision 27874) +++ /trunk/pstamp/scripts/pstamp_cleanup.pl (revision 27874) @@ -0,0 +1,201 @@ +#!/bin/env perl +### +### pstamp_cleanup.pl +### delete all data associated with a postage stamp request +### +### This script should probably be called request_cleanup.pl since it handles more than postage stamp requests +### + +use warnings; +use strict; + +use Sys::Hostname; +use Getopt::Long qw( GetOptions ); +use File::Basename qw( basename dirname); +use File::Copy; +use POSIX qw( strftime ); +use Carp; +use IPC::Cmd 0.36 qw( can_run run ); + +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw( :standard ); + +my $req_id; +my $uri; +my $name; +my $outdir; +my $reqType; +my $redirect_output; +my $product; +my $label; +my $verbose; +my $dbname; +my $dbserver; +my $no_update; + +GetOptions( + 'req_id=s' => \$req_id, + 'name=s' => \$name, + 'outdir=s' => \$outdir, + 'reqType=s' => \$reqType, + 'product=s' => \$product, + 'uri=s' => \$uri, + 'outdir=s' => \$outdir, + 'redirect-output' => \$redirect_output, + 'verbose' => \$verbose, + 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, + 'no-update' => \$no_update, +); + +if ($verbose) { + my $host = hostname(); + print "\n\n"; + print "Starting script $0 on $host\n\n"; +} + +my $missing_tools; + +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + + +die("--req_id --name --product and --outdir are required") if !defined($req_id) or !defined($product) or !defined($name) or !defined $outdir; + +my $ipprc = PS::IPP::Config->new(); # IPP Configuration + +my $pstamp_workdir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_WORKDIR'); +exit ($PS_EXIT_CONFIG_ERROR) unless defined $pstamp_workdir; # lookup failure outputs a message + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + +my_die("Cleanup not yet supported for reqType: $reqType", $req_id, $PS_EXIT_UNKNOWN_ERROR) + if ($reqType ne "pstamp") and ($reqType ne "NULL"); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +if ($product ne 'NULL' and $name ne 'NULL') { + my $command = "$dsreg --product $product --del $name --rm --force"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("Unable to perform $command error code: $error_code", $req_id, $PS_EXIT_UNKNOWN_ERROR); + } +} + +# now go find the workdir for this request +# XXX: we finally *have* to store this in the database +$outdir = undef if $outdir eq "NULL"; + +if ($outdir) { + delete_workdir($outdir); +} else { + if ($uri ne 'NULL') { + # use the URI to find the workdir + $outdir = dirname($uri); + if (index($outdir, $pstamp_workdir) == 0) { + print "pstamp workdir for req_id $req_id is $outdir\n"; + delete_workdir($outdir); + } else { + print "no outdir found for $req_id\n"; + } + } + if (!$outdir) { + # gotta go look for it in the subdirectories of $pstamp_workdir + my $command = "find $pstamp_workdir -maxdepth 2 -type d -name $req_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my $rc = $error_code >> 8; + my_die("Unable to perform $command error code: $error_code", $req_id, $rc); + } + if ($stdout_buf) { + # for historical reasons (a bug) there may be more than one outdir + my @outdirs = split "\n", join "", $stdout_buf; + foreach $outdir (@outdirs) { + chomp $outdir; + delete_workdir($outdir); + } + } else { + print "no outdir found for $req_id\n"; + } + } +} + + +{ + my $command = "$pstamptool -updatereq -req_id $req_id -set_state cleaned"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + die("Unable to perform $command error code: $error_code"); + } + } else { + print "skipping $command\n"; + } +} + +exit 0; + + +sub delete_workdir { + my $dir = shift; + + if (!-e $dir) { + print "outdir $dir does not exist\n"; + return 0; + } + if (!-d $dir) { + print "outdir $dir is not a directory\n"; + return 0; + } + + my $command = "rm -r $dir"; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my $rc = $error_code >> 8; + my_die("Unable to perform $command return status: $rc", $req_id, $rc); + } + } else { + print "skipping $command\n"; + } + return 0; +} + +sub my_die { + my $msg = shift; + my $req_id = shift; + my $fault = shift; + + carp($msg); + + my $command = "$pstamptool -updatereq -req_id $req_id -set_fault $fault"; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + die("Unable to perform $command error code: $error_code"); + } + } else { + print "skipping $command\n"; + } + exit $fault; +} Index: /trunk/pstamp/scripts/pstamp_dorequest.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_dorequest.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_dorequest.pl (revision 27874) @@ -106,5 +106,5 @@ { ## TODO: what about request status - my $command = "$pstamptool -updatereq -req_id $request_id -state stop"; + my $command = "$pstamptool -updatereq -req_id $request_id -set_state stop"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbosity); Index: /trunk/pstamp/scripts/pstamp_finish.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_finish.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_finish.pl (revision 27874) @@ -25,5 +25,5 @@ use PS::IPP::PStamp::Job qw( :standard ); -my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); +my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); # the char to the right of the bar may be used as a single - alias for the longer name @@ -34,5 +34,5 @@ 'req_file=s' => \$req_file, 'product=s' => \$product, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'dbname=s' => \$dbname, 'dbserver=s' => \$dbserver, @@ -44,10 +44,10 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n" - if !$req_id or !$req_name or !$req_file or !$product or !$out_dir; +die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n" + if !$req_id or !$req_name or !$req_file or !$product or !$outdir; my $ipprc = PS::IPP::Config->new(); # IPP Configuration if ($redirect_output) { - my $logDest = "$out_dir/psfinish.$req_id.log"; + my $logDest = "$outdir/psfinish.$req_id.log"; $ipprc->redirect_output($logDest); } @@ -82,20 +82,20 @@ my $fileset = $req_name; - print STDERR "product: $product REQ_NAME: $req_name $out_dir\n" if $verbose; - - if (!-e $out_dir) { + print STDERR "product: $product REQ_NAME: $req_name $outdir\n" if $verbose; + + if (!-e $outdir) { # something must have gone wrong parsing the request - print STDERR "output directory $out_dir does not exist\n"; - - if (!mkdir $out_dir) { - print STDERR "cannot create output directory $out_dir"; + print STDERR "output directory $outdir does not exist\n"; + + if (!mkdir $outdir) { + print STDERR "cannot create output directory $outdir"; stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } - } elsif (! -d $out_dir ) { + } elsif (! -d $outdir ) { # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the # request - print STDERR "output directory $out_dir exists but is not a directory"; + print STDERR "output directory $outdir exists but is not a directory"; stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -122,14 +122,14 @@ # the following is for a postage stamp request - my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps); + my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps); print $rlf "results.fits|||table|\n"; print $rlf "results.mdc|||text|\n"; my $err_file = "parse_error.txt"; - if (-e "$out_dir/$err_file" ) { + if (-e "$outdir/$err_file" ) { print $rlf "$err_file|||text|\n"; } - my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps); + my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps); @@ -208,5 +208,5 @@ if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) { - my $jreglist = "$out_dir/reglist$job_id"; + my $jreglist = "$outdir/reglist$job_id"; if (open JRL, "<$jreglist") {; # process the reglist file to get the list of files produced by this job @@ -225,7 +225,7 @@ # If not found check the PHU. If that doesn't work just set them to zero. # XXX do this more cleanly - my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`; + my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`; if (!defined $ra_deg) { - (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`; + (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`; } $ra_deg = 0.0 if (!$ra_deg); @@ -258,5 +258,5 @@ # make the results file { - my $command = "$pstamp_results --input $table_def_name --output $out_dir/results.fits"; + my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -266,5 +266,5 @@ } else { # dump a textual representation - my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc"; + my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -278,6 +278,7 @@ # register the fileset my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS"; - $command .= " --link --datapath $out_dir --ps0 $req_id"; - $command .= " --dbname $dbname" if $dbname; + $command .= " --link --datapath $outdir --ps0 $req_id"; +# XXX: let dsreg and config handle resolving dbname and dbserver +# $command .= " --dbname $dbname" if $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -292,5 +293,5 @@ # set the request's state to stop { - my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -308,5 +309,5 @@ my $fault = shift; - my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; Index: /trunk/pstamp/scripts/pstamp_get_image_job.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_get_image_job.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_get_image_job.pl (revision 27874) @@ -87,5 +87,5 @@ } -my $out_dir = dirname($output_base); +my $outdir = dirname($output_base); my $prefix = basename($output_base) . "_"; my $results_file = $output_base . ".bundle_results"; @@ -102,5 +102,5 @@ my $command = "$dist_bundle --camera $camera --stage $stage --stage_id $stage_id"; $command .= " --component $component"; - $command .= " --path_base $path_base --outdir $out_dir --results_file $results_file"; + $command .= " --path_base $path_base --outdir $outdir --results_file $results_file"; $command .= " --prefix $prefix"; $command .= " --magicked" if $magicked; @@ -139,5 +139,5 @@ } -my $reglist = "$out_dir/reglist$job_id"; +my $reglist = "$outdir/reglist$job_id"; if (! open(REGLIST, ">$reglist") ) { my_die("failed to open registration list: $reglist", $PS_EXIT_UNKNOWN_ERROR); Index: /trunk/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_job_run.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_job_run.pl (revision 27874) @@ -198,6 +198,6 @@ # mark the job stopped in the database { - my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; - $command .= " -fault $jobStatus" if $jobStatus; + my $command = "$pstamptool -updatejob -job_id $job_id -set_state stop"; + $command .= " -set_fault $jobStatus" if $jobStatus; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -343,7 +343,7 @@ my $command = "$pstamptool -updatejob"; $command .= " -job_id $job_id"; - $command .= " -fault $exit_code"; + $command .= " -set_fault $exit_code"; # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided - $command .= " -state run"; + $command .= " -set_state run"; $command .= " -dbname $dbname" if defined $dbname; $command .= " -dbserver $dbserver" if defined $dbserver; Index: /trunk/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27873) +++ /trunk/pstamp/scripts/pstamp_parser_run.pl (revision 27874) @@ -28,4 +28,5 @@ my $redirect_output; my $product; +my $outdir; my $label; my $verbose; @@ -37,4 +38,5 @@ 'uri=s' => \$uri, 'product=s' => \$product, + 'outdir=s' => \$outdir, 'label=s' => \$label, 'redirect-output' => \$redirect_output, @@ -81,21 +83,25 @@ } -# workdir is where all of the files generated for this request are placed -# NOTE: this location needs to be kept in sync with the web interface ( request.php ) -my $datestr = strftime "%Y%m%d", gmtime; -my $datedir = "$pstamp_workdir/$datestr"; -if (! -e $datedir ) { - mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id, +if (!$outdir or ($outdir eq "NULL")) { + # workdir is where all of the files generated for this request are placed + # NOTE: this location needs to be kept in sync with the web interface ( request.php ) + my $datestr = strftime "%Y%m%d", gmtime; + my $datedir = "$pstamp_workdir/$datestr"; + if (! -e $datedir ) { + mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id, + $PS_EXIT_CONFIG_ERROR); + } + + $outdir = "$datedir/$req_id"; +} + +if (! -e $outdir ) { + mkdir $outdir or my_die("failed to create working directory $outdir for request id $req_id", $req_id, $PS_EXIT_CONFIG_ERROR); } - -my $workdir = "$datedir/$req_id"; -if (! -e $workdir ) { - mkdir $workdir or my_die("failed to create working directory $workdir for request id $req_id", $req_id, - $PS_EXIT_CONFIG_ERROR); -} + if ($redirect_output) { - my $logDest = "$workdir/psparse.$req_id.log"; + my $logDest = "$outdir/psparse.$req_id.log"; $ipprc->redirect_output($logDest); } @@ -108,5 +114,5 @@ my $fn = basename($uri); -my $new_uri = "$workdir/$fn"; +my $new_uri = "$outdir/$fn"; if ($uri =~ /^http:/) { # if the uri is an http uri download the file @@ -123,5 +129,5 @@ } if (! copy $uri, $new_uri) { - my_die ("failed to copy request file $uri to workdir $workdir", $req_id, $PS_EXIT_UNKNOWN_ERROR); + my_die ("failed to copy request file $uri to workdir $outdir", $req_id, $PS_EXIT_UNKNOWN_ERROR); } } @@ -173,7 +179,7 @@ # that they sent us a request file that we don't understand - my $command = "$pstamptool -updatereq -req_id $req_id -state run"; - $command .= " -reqType unknown"; - $command .= " -fault $PS_EXIT_DATA_ERROR"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state run"; + $command .= " -set_reqType unknown"; + $command .= " -set_fault $PS_EXIT_DATA_ERROR"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -186,5 +192,5 @@ } -$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri"; +$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --outdir $outdir --file $uri"; $parse_cmd .= " --dbname $dbname" if $dbname; $parse_cmd .= " --dbserver $dbserver" if $dbserver; @@ -194,5 +200,5 @@ my $fault; { - my $error_file_name = "$workdir/parse_error.txt"; + my $error_file_name = "$outdir/parse_error.txt"; # get rid of any error file from previous attempt to parse this request unlink $error_file_name if (-e $error_file_name); @@ -227,8 +233,9 @@ # { - my $command = "$pstamptool -updatereq -req_id $req_id -state $newState"; - $command .= " -reqType $reqType" if $reqType; - $command .= " -uri $new_uri" if $new_uri; - $command .= " -fault $fault" if $fault; + my $command = "$pstamptool -updatereq -req_id $req_id -set_state $newState"; + $command .= " -set_outdir $outdir"; + $command .= " -set_reqType $reqType" if $reqType; + $command .= " -set_uri $new_uri" if $new_uri; + $command .= " -set_fault $fault" if $fault; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; @@ -269,5 +276,5 @@ carp($msg); - my $command = "$pstamptool -updatereq -req_id $req_id -fault $fault"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_fault $fault"; $command .= " -dbname $dbname" if $dbname; $command .= " -dbserver $dbserver" if $dbserver; Index: /trunk/pstamp/scripts/pstampparse.pl =================================================================== --- /trunk/pstamp/scripts/pstampparse.pl (revision 27873) +++ /trunk/pstamp/scripts/pstampparse.pl (revision 27874) @@ -25,5 +25,5 @@ my $request_file_name; my $mode = "list_uri"; -my $out_dir; +my $outdir; my $product; my $label; @@ -34,5 +34,5 @@ 'file=s' => \$request_file_name, 'req_id=s' => \$req_id, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'product=s' => \$product, 'label=s' => \$label, @@ -50,5 +50,5 @@ if ($mode ne "list_uri") { die "req_id is required" if !$req_id; - die "out_dir is required" if !$out_dir; + die "outdir is required" if !$outdir; die "product is required" if !$product; } @@ -129,6 +129,6 @@ # update the database with the request name. This will be used as the # the output data store's product name - my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; - $command .= " -outProduct $product"; + my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name"; + $command .= " -set_outProduct $product"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -522,5 +522,5 @@ $base =~ s/.fits$//; - my $output_base = "$out_dir/${rownum}_${job_num}_${base}"; + my $output_base = "$outdir/${rownum}_${job_num}_${base}"; my $argslist = "${output_base}.args"; @@ -712,5 +712,5 @@ my $exp_id = $image->{exp_id}; - my $output_base = "$out_dir/${rownum}_${job_num}"; + my $output_base = "$outdir/${rownum}_${job_num}"; write_params($output_base, $image); @@ -894,5 +894,5 @@ my $dep_id; - my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $out_dir"; + my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir"; $command .= " -need_magic" if $need_magic; Index: /trunk/pstamp/scripts/request_finish.pl =================================================================== --- /trunk/pstamp/scripts/request_finish.pl (revision 27873) +++ /trunk/pstamp/scripts/request_finish.pl (revision 27874) @@ -21,5 +21,5 @@ use PS::IPP::Config qw( :standard ); -my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output ); +my ( $req_id, $req_name, $req_file, $req_type, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output ); GetOptions( @@ -28,5 +28,5 @@ 'req_file=s' => \$req_file, 'req_type=s' => \$req_type, - 'out_dir=s' => \$out_dir, + 'outdir=s' => \$outdir, 'product=s' => \$product, 'dbname=s' => \$dbname, @@ -46,15 +46,15 @@ $err .= "--req_name is required\n" if !$req_name; $err .= "--product is required\n" if !$product; -# $err .= "--out_dir is required\n" if !$out_dir; +# $err .= "--outdir is required\n" if !$outdir; die "$err" if $err; -if (!$out_dir) { - $out_dir = dirname($req_file); +if (!$outdir) { + $outdir = dirname($req_file); } if ($redirect_output) { - my $logDest = "$out_dir/reqfinish.$req_id.log"; + my $logDest = "$outdir/reqfinish.$req_id.log"; my $ipprc = PS::IPP::Config->new(); $ipprc->redirect_output($logDest); @@ -79,5 +79,5 @@ } if ($finish_cmd) { - my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir"; + my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --outdir $outdir"; $command .= " --dbname $dbname" if $dbname; $command .= " --dbserver $dbserver" if $dbserver;