Index: /branches/czw_branch/20101203/ippTasks/register.pro =================================================================== --- /branches/czw_branch/20101203/ippTasks/register.pro (revision 29907) +++ /branches/czw_branch/20101203/ippTasks/register.pro (revision 29908) @@ -8,8 +8,10 @@ book init regPendingImfile book init regPendingExp +book init regPendingBurntoolImfile macro register.reset book init regPendingImfile book init regPendingExp + book init regPendingBurntoolImfile end @@ -18,4 +20,5 @@ book listbook regPendingImfile book listbook regPendingExp + book listbook regPendingBurntoolImfile end @@ -33,4 +36,10 @@ active true end + task register.burntool.load + active true + end + task register.burntool.run + active true + end end @@ -46,4 +55,10 @@ end task register.exp.run + active false + end + task register.burntool.load + active false + end + task register.burntool.run active false end @@ -429,3 +444,130 @@ end + +# select imfiles that can now be burntooled. +task register.burntool.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGDIR/register.burntool.load.log + + # select entried from the current DB; cycle to the next DB, if it exists + task.exec + $run = regtool -pendingburntoolimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$regPendingBurntoolImfile_DB + $run = $run -dbname $DB:$regPendingBurntoolImfile_DB + $regPendingBurntoolImfile_DB ++ + if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0 + end + + $today = `date +%Y-%m-%d` + $run = $run -date $today -valid_burntool $valid_burntool_value + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingBurntoolImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle regPendingBurntoolImfile + if ($VERBOSE > 2) + book listbook regPendingBurntoolImfile + end + + # delete existing entries in the appropriate pantasksStates + process_cleanup regPendingBurntoolImfile + end + + # locked list + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the ipp_apply_burntool_single.pl script on the pending images +task register.burntool.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages regPendingBurntoolImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images to burn in regPendingBurntoolImfile + book getpage regPendingBurntoolImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingBurntoolImfile $pageName pantaskState RUN + + book getword regPendingBurntoolImfile $pageName exp_id -var EXP_ID + book getword regPendingBurntoolImfile $pageName class_id -var CLASS_ID + book getword regPendingBurntoolImfile $pageName uri -var THIS_URI + book getword regPendingBurntoolImfile $pageName previous_uri -var PREVIOUS_URI + book getword regPendingBurntoolImfile $pageName camera -var CAMERA + book getword regPendingBurntoolImfile $pageName dbname -var DBNAME + set.host.for.camera $CAMERA $CLASS_ID + + stdout $LOGDIR/register.burntool.run.log + stderr $LOGDIR/register.burntool.run.log + + $run = ipp_apply_burntool_single.pl --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI + if ("$PREVIOUS_URI" != "NULL") + $run = $run --previous_uri $PREVIOUS_URI + end + + add_standard_args run + + options $pageName + + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingBurntoolImfile $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword regPendingBurntoolImfile $options:0 pantaskState CRASH + end + + # operation timed out + task.exit timeout + showcommand timeout + book setword regPendingBurntoolImfile $options:0 pantaskState TIMEOUT + end +end + + + + ## XXX add a global path to output files Index: /branches/czw_branch/20101203/ippTools/share/Makefile.am =================================================================== --- /branches/czw_branch/20101203/ippTools/share/Makefile.am (revision 29907) +++ /branches/czw_branch/20101203/ippTools/share/Makefile.am (revision 29908) @@ -316,4 +316,5 @@ receivetool_revert.sql \ receivetool_toadvance.sql \ + regtool_checkburntoolimfile.sql \ regtool_create_dup_table.sql \ regtool_export_exp.sql \ @@ -331,4 +332,5 @@ regtool_updatebyquery.sql \ regtool_updatebyqueryimfile.sql \ + regtool_pendingburntoolimfile.sql \ stacktool_associationdefine_select.sql \ stacktool_definebyquery_insert.sql \ Index: /branches/czw_branch/20101203/ippTools/share/regtool_checkburntoolimfile.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/regtool_checkburntoolimfile.sql (revision 29908) +++ /branches/czw_branch/20101203/ippTools/share/regtool_checkburntoolimfile.sql (revision 29908) @@ -0,0 +1,31 @@ +SELECT DISTINCT + exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + LEFT JOIN pzDownloadExp USING(exp_name) + LEFT JOIN newExp ON exp_name = tmp_exp_name + LEFT JOIN rawExp USING(exp_id, exp_name) + LEFT JOIN rawImfile USING(exp_name) + WHERE + summitExp.dateobs >= '@DATE@T00:00:00' + AND summitExp.dateobs <= '@DATE@T23:59:59' + AND (rawImfile.class_id = '@CLASS_ID@' OR rawImfile.class_id IS NULL) + AND (exp_name <= '@EXP_NAME@') +ORDER BY summitExp.dateobs Index: /branches/czw_branch/20101203/ippTools/share/regtool_pendingburntoolimfile.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/regtool_pendingburntoolimfile.sql (revision 29908) +++ /branches/czw_branch/20101203/ippTools/share/regtool_pendingburntoolimfile.sql (revision 29908) @@ -0,0 +1,31 @@ +SELECT DISTINCT + summitExp.exp_name, + rawImfile.uri, + summitExp.dateobs AS registered, + summitExp.imfiles, + summitExp.fault AS summit_fault, + pzDownloadExp.state AS download_state, + newExp.state AS newExp_state, + rawExp.state AS rawExp_state, + rawImfile.data_state AS imfile_state, + newExp.exp_id, + newImfile.tmp_class_id, + summitExp.exp_type, + rawExp.dateobs, + rawExp.obs_mode AS obs_mode, + rawExp.object AS object, + rawImfile.burntool_state AS burntool_state, + rawImfile.class_id, + (pzDownloadExp.state = 'stop') AS is_downloaded, + (rawImfile.burntool_state IS NOT NULL) AS is_registered + FROM + summitExp + LEFT JOIN pzDownloadExp USING(exp_name) + LEFT JOIN newExp ON exp_name = tmp_exp_name + LEFT JOIN newImfile USING(exp_id) + LEFT JOIN rawExp USING(exp_id, exp_name) + LEFT JOIN rawImfile ON (rawImfile.exp_name = rawExp.exp_name AND rawImfile.tmp_class_id = newImfile.tmp_class_id) + WHERE + summitExp.dateobs >= '@DATE@T00:00:00' + AND summitExp.dateobs <= '@DATE@T23:59:59' +ORDER BY newImfile.tmp_class_id,summitExp.dateobs Index: /branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 29907) +++ /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 29908) @@ -34,4 +34,6 @@ static bool pendingimfileMode(pxConfig *config); static bool addprocessedimfileMode(pxConfig *config); +static bool checkburntoolimfileMode(pxConfig *config); +static bool pendingburntoolimfileMode(pxConfig *config); static bool processedimfileMode(pxConfig *config); static bool revertprocessedimfileMode(pxConfig *config); @@ -72,4 +74,6 @@ // imfile MODECASE(REGTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileMode); + MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode); MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileMode); MODECASE(REGTOOL_MODE_PROCESSEDIMFILE, processedimfileMode); @@ -176,4 +180,249 @@ } +static bool checkburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psString query = pxDataGet("regtool_checkburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,exp_name,"@EXP_NAME@"); + psStringSubstitute(&query,class_id,"@CLASS_ID@"); + psStringSubstitute(&query,date,"@DATE@"); + + fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + ok_to_burn = false; + } + if (already_burned == false) { + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + psFree(previous_uri); + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + return(true); + +} + +static bool pendingburntoolimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config,false); + + // required + PXOPT_LOOKUP_STR(date, config->args, "-date", true, false); + PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false); + // optional + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false); + psString query = pxDataGet("regtool_pendingburntoolimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + psString rep = psStringCopy(query); + psFree(query); + query = rep; + + psStringSubstitute(&query,date,"@DATE@"); + + fprintf(stderr,"%s",query); + + if (!p_psDBRunQuery(config->dbh, query)) { + // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms) + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + psString previous_uri = NULL; + bool ok_to_burn = true; + bool already_burned = true; + psString previous_class_id = NULL; + psString this_class_id = NULL; + psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool. + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + + // Use tmp_class_id because class_id may not be populated for all imfiles + this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&this_class_id,"XY","ota"); + // Determine if we've crossed a class_id boundary. + if (previous_class_id) { + if (strcmp(this_class_id,previous_class_id) != 0) { + ok_to_burn = true; + already_burned = true; + previous_uri = NULL; + } + } + + // If the state of this imfile is not "pending_burntool" then skip it, because + if (!ignore_state) { + psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state"); + + if (!imfile_state) { + continue; + } + if (strcmp("pending_burntool",imfile_state) != 0) { + continue; + } + } + + // Convert bits of the SQL query into booleans describing the data state + if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| + (psMetadataLookupS32(NULL,row,"is_registered") != 1)) { + ok_to_burn = false; + } + if (already_burned == false) { + ok_to_burn = false; + } + if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) { + already_burned = true; + } + else { + already_burned = false; + } + psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn); + psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned); + + // We'll need to return the previous uri so we know where the input table is: + if (previous_uri) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri); + psFree(previous_uri); + } + previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); + + // Write the class_id stuff at the end for debugging. + psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id); + if (previous_class_id) { + psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id); + psFree(previous_class_id); + } + // Use tmp_class_id because class_id may not be populated for all imfiles + previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id")); + psStringSubstitute(&previous_class_id,"XY","ota"); + // Determine if we've already suggested an entry for this ota, and if not, copy this + // suggestion into our output result list. + if (!ok_to_burn || already_burned) { + continue; + } + + psArrayAdd(results,results->n,row); + // If we're here, then we think we could potentially burntool this file. + + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + psFree(results); + return false; + } + + psFree(output); + psFree(results); + + + return(true); + +} + + static bool addprocessedimfileMode(pxConfig *config) Index: /branches/czw_branch/20101203/ippTools/src/regtool.h =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtool.h (revision 29907) +++ /branches/czw_branch/20101203/ippTools/src/regtool.h (revision 29908) @@ -26,4 +26,6 @@ REGTOOL_MODE_NONE = 0x0, REGTOOL_MODE_PENDINGIMFILE, + REGTOOL_MODE_CHECKBURNTOOLIMFILE, + REGTOOL_MODE_PENDINGBURNTOOLIMFILE, REGTOOL_MODE_ADDPROCESSEDIMFILE, REGTOOL_MODE_PROCESSEDIMFILE, Index: /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 29907) +++ /branches/czw_branch/20101203/ippTools/src/regtoolConfig.c (revision 29908) @@ -53,4 +53,22 @@ ADD_OPT(Bool, pendingimfileArgs, "-simple", "use the simple output format", false); + // -checkburntoolimfile + psMetadata *checkburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, checkburntoolimfileArgs, "-exp_name", "define the exp_name (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-class_id", "define class ID (required)", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Str, checkburntoolimfileArgs, "-inst", "define the camera name", NULL); + ADD_OPT(Str, checkburntoolimfileArgs, "-telescope", "define the telescope name", NULL); + ADD_OPT(Bool, checkburntoolimfileArgs, "-simple", "use the simple output format", false); + + // -pendingburntoolimfile + psMetadata *pendingburntoolimfileArgs = psMetadataAlloc(); + ADD_OPT(Str, pendingburntoolimfileArgs, "-date", "select the date to process (required)", NULL); + ADD_OPT(S32, pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple", "use the simple output format", false); + ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state", "ignore the data_state when deciding what to work on", false); + ADD_OPT(U64, pendingburntoolimfileArgs, "-limit", "limit result set to N items", 0); + // -addprocessedimfile psMetadata *addprocessedimfileArgs = psMetadataAlloc(); @@ -395,4 +413,6 @@ PXOPT_ADD_MODE("-pendingimfile", "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + PXOPT_ADD_MODE("-checkburntoolimfile", "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs); + PXOPT_ADD_MODE("-pendingburntoolimfile", "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs); PXOPT_ADD_MODE("-addprocessedimfile", "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs); PXOPT_ADD_MODE("-processedimfile", "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);