- Timestamp:
- Mar 16, 2010, 6:28:29 PM (16 years ago)
- Location:
- trunk
- Files:
-
- 6 edited
-
ippTasks/dqstats.pro (modified) (10 diffs)
-
ippTools/share/dqstatstool_definebyquery.sql (modified) (2 diffs)
-
ippTools/share/dqstatstool_get_run.sql (modified) (1 diff)
-
ippTools/src/dqstatstool.c (modified) (15 diffs)
-
ippTools/src/dqstatstoolConfig.c (modified) (1 diff)
-
ippconfig/recipes/dqStatsTool.config (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippTasks/dqstats.pro
r26378 r27304 1 1 ## dqstats.pro : globals and support macros : -*- sh -*- 2 2 ## this file contains the tasks for running the dqstats analysis stage 3 ## these tasks use the book dqstatsPending Exp3 ## these tasks use the book dqstatsPendingBundle 4 4 5 5 # test for required global variables 6 6 check.globals 7 7 8 book init dqstatsPending Exp8 book init dqstatsPendingBundle 9 9 10 10 macro dqstats.status 11 book listbook dqstatsPending Exp11 book listbook dqstatsPendingBundle 12 12 end 13 13 14 14 macro dqstats.reset 15 book init dqstatsPending Exp15 book init dqstatsPendingBundle 16 16 end 17 17 … … 45 45 46 46 # select images ready for dqstats analysis 47 # new entries are added to dqstatsPending Exp47 # new entries are added to dqstatsPendingBundle 48 48 # skip already-present entries 49 49 task dqstats.load … … 70 70 if ($dqstats_DB >= $DB:n) set dqstats_DB = 0 71 71 end 72 add_poll_args run73 add_poll_labels run72 # add_poll_args run 73 # add_poll_labels run 74 74 command $run 75 75 end … … 78 78 task.exit 0 79 79 # convert 'stdout' to book format 80 ipptool2book stdout dqstatsPending Exp-key dqstats_id -uniq -setword dbname $options:0 -setword pantaskState INIT80 ipptool2book stdout dqstatsPendingBundle -key dqstats_id -uniq -setword dbname $options:0 -setword pantaskState INIT 81 81 if ($VERBOSE > 2) 82 book listbook dqstatsPending Exp82 book listbook dqstatsPendingBundle 83 83 end 84 84 85 85 # delete existing entries in the appropriate pantaskStates 86 process_cleanup dqstatsPending Exp86 process_cleanup dqstatsPendingBundle 87 87 end 88 88 … … 106 106 periods -poll $RUNPOLL 107 107 periods -exec $RUNEXEC 108 periods -timeout 10108 periods -timeout 600 109 109 110 110 ## we want only a single outstanding dqstats job. … … 113 113 114 114 task.exec 115 book npages dqstatsPending Exp-var N115 book npages dqstatsPendingBundle -var N 116 116 if ($N == 0) break 117 117 if ($NETWORK == 0) break 118 118 119 # look for new images in dqstatsPending Exp(pantaskState == INIT)120 book getpage dqstatsPending Exp0 -var pageName -key pantaskState INIT119 # look for new images in dqstatsPendingBundle (pantaskState == INIT) 120 book getpage dqstatsPendingBundle 0 -var pageName -key pantaskState INIT 121 121 if ("$pageName" == "NULL") break 122 122 123 book setword dqstatsPendingExp $pageName pantaskState RUN 124 book getword dqstatsPendingExp $pageName camera -var CAMERA 125 book getword dqstatsPendingExp $pageName exp_tag -var EXP_TAG 126 book getword dqstatsPendingExp $pageName dqstats_id -var DQSTATS_ID 127 # book getword dqstatsPendingExp $pageName camroot -var CAMROOT 128 # book getword dqstatsPendingExp $pageName workdir -var WORKDIR_TEMPLATE 129 # book getword dqstatsPendingExp $pageName reduction -var REDUCTION 130 # book getword dqstatsPendingExp $pageName dvodb -var DVODB 131 # book getword dqstatsPendingExp $pageName image_only -var IMAGE_ONLY 132 book getword dqstatsPendingExp $pageName dbname -var DBNAME 123 book setword dqstatsPendingBundle $pageName pantaskState RUN 124 book getword dqstatsPendingBundle $pageName camera -var CAMERA 125 book getword dqstatsPendingBundle $pageName dqstats_id -var DQSTATS_ID 126 book getword dqstatsPendingBundle $pageName dbname -var DBNAME 133 127 134 128 # specify choice of remote host based on camera and chip (class_id) … … 153 147 stderr $LOGDIR/dqstats.log 154 148 155 $run = dqstats_bundle.pl --exp_tag $EXP_TAG --dqstats_id $DQSTATS_ID --camera GPC1 --redirect-output 156 # if ("$REDUCTION" != "NULL") 157 # $run = $run --reduction $REDUCTION 158 # end 159 # if ("$IMAGE_ONLY" == "T") 160 # $run = $run --image-only 161 # end 162 149 $run = dqstats_bundle.pl --dqstats_id $DQSTATS_ID --camera $CAMERA --redirect-output 163 150 164 151 add_standard_args run … … 176 163 # success 177 164 task.exit default 178 process_exit dqstatsPending Exp$options:0 $JOB_STATUS165 process_exit dqstatsPendingBundle $options:0 $JOB_STATUS 179 166 end 180 167 … … 183 170 showcommand crash 184 171 echo "hostname: $JOB_HOSTNAME" 185 book setword dqstatsPending Exp$options:0 pantaskState CRASH172 book setword dqstatsPendingBundle $options:0 pantaskState CRASH 186 173 end 187 174 … … 189 176 task.exit timeout 190 177 showcommand timeout 191 book setword dqstatsPending Exp$options:0 pantaskState TIMEOUT178 book setword dqstatsPendingBundle $options:0 pantaskState TIMEOUT 192 179 end 193 180 end -
trunk/ippTools/share/dqstatstool_definebyquery.sql
r26378 r27304 1 1 -- query to identify unique data to bundle 2 2 3 SELECT exp_id,chip_id,cam_id,warp_id,invalid from rawExp3 SELECT DISTINCT exp_id,chip_id,cam_id,warp_id,invalid from rawExp 4 4 JOIN chipRun USING(exp_id) 5 5 JOIN camRun USING(chip_id) … … 7 7 LEFT OUTER JOIN warpRun USING(fake_id) 8 8 LEFT OUTER JOIN dqstatsContent USING (exp_id,chip_id,cam_id,warp_id) 9 WHERE (invalid IS NULL OR invalid = 1)9 WHERE camRun.state = 'full' -
trunk/ippTools/share/dqstatstool_get_run.sql
r26378 r27304 1 SELECT * from dqstatsRun 2 WHERE state = 'new' AND (fault = 0) 1 SELECT DISTINCT dqstatsRun.*,rawExp.camera from dqstatsRun 2 JOIN dqstatsContent USING(dqstats_id) 3 JOIN rawExp USING(exp_id) 4 WHERE dqstatsRun.state = 'new' AND (dqstatsRun.fault = 0) -
trunk/ippTools/src/dqstatstool.c
r26981 r27304 45 45 } 46 46 47 psFree(config);47 //psFree(config); 48 48 pmConfigDone(); 49 49 psLibFinalize(); … … 54 54 int exit_status = pxerrorGetExitStatus(); 55 55 56 psFree(config);56 // psFree(config); 57 57 pmConfigDone(); 58 58 psLibFinalize(); … … 88 88 } 89 89 // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE 90 if (force) { 91 psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1 OR invalid = 0) "); 92 } 93 else { 94 psStringAppend(&query, " AND (invalid IS NULL OR invalid = 1) "); 95 } 96 90 97 if (where && psListLength(where->list)) { 91 98 psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); 92 99 psStringAppend(&query, " AND %s", whereClause); 93 100 psFree(whereClause); 94 }95 if (force) {96 psStringAppend(&query, " OR invalid = 0");97 101 } 98 102 … … 103 107 psFree(limitString); 104 108 } 105 109 psTrace("debug",3,">> %s",query); 106 110 if (!p_psDBRunQuery(config->dbh, query)) { 107 111 psError(PS_ERR_UNKNOWN, false, "database error"); … … 325 329 psFree(output); // This held the information about the Run. 326 330 327 // Read column list from config file .331 // Read column list from config file 328 332 psMetadata *recipe = psMetadataLookupPtr(&status, config->modules->recipes, "DQSTATS"); 329 333 psMetadata *columns = psMetadataLookupMetadata(&status,recipe,"STATS.COLUMNS"); … … 342 346 psMetadata *colTypes = psMetadataAlloc(); 343 347 while ((colItem = psMetadataGetAndIncrement(iter))) { 348 if (strcmp(colItem->name, "COLUMN")) continue; 349 if (colItem->type != PS_DATA_METADATA) { 350 psError(PS_ERR_UNKNOWN, false, "dqstatstool COLUMN not of type METADATA\n"); 351 return(NULL); 352 } 353 344 354 bool status; 345 355 psMetadata *coldef = colItem->data.md; … … 369 379 return(false); 370 380 } 381 382 // Read decision engine settings from config file. 383 psMetadata *engine = psMetadataLookupMetadata(&status,recipe,"DECISION.ENGINE"); 384 if (!engine) { 385 psError(PS_ERR_UNKNOWN, false, "Could not read DECISION.ENGINE metadata"); 386 psFree(whereClause); 387 return(false); 388 } 389 psArray *decisionRules = psArrayAllocEmpty(4); // list of screened and verified rules. 390 psMetadataIterator *iterEngine = psMetadataIteratorAlloc(engine, PS_LIST_HEAD,NULL); 391 psMetadataItem *engineItem = NULL; 392 393 while ((engineItem = psMetadataGetAndIncrement(iterEngine))) { 394 if (strcmp(engineItem->name, "RULE")) continue; 395 if (engineItem->type != PS_DATA_METADATA) { 396 psError(PS_ERR_UNKNOWN, false, "dqstatstool RULE not of type METADATA\n"); 397 return(NULL); 398 } 399 400 bool status; 401 psMetadata *enginedef = engineItem->data.md; 402 403 if (psMetadataLookupStr(&status,enginedef,"COLNAME")&& 404 isfinite(psMetadataLookupF32(&status,enginedef,"MINIMUM"))&& 405 isfinite(psMetadataLookupF32(&status,enginedef,"MAXIMUM"))) { 406 psArrayAdd(decisionRules,decisionRules->n,enginedef); 407 } 408 else { 409 psError(PS_ERR_UNKNOWN, false, "Rule invalid."); 410 } 411 } 412 psFree(iterEngine); 413 414 if (decisionRules->n < 1) { 415 psError(PS_ERR_UNKNOWN, false, "Failed to generate a decision engine"); 416 psFree(decisionRules); 417 psFree(engine); 418 return(false); 419 } 420 psFree(engine); 371 421 372 422 // Find the contents that comprise this run … … 439 489 bool status; 440 490 psString MYquery = psStringCopy(query); 491 if (!MYquery) { 492 psTrace("debug",4,"Failed to copy, trying again."); 493 MYquery = psStringCopy(query); 494 } 495 441 496 psMetadata *where = psMetadataAlloc(); 442 497 … … 445 500 psMetadataAddS64(where,PS_LIST_TAIL, "camRun.cam_id", 0, "==", psMetadataLookupS64(&status,row,"cam_id")); 446 501 // This may not be real, so we need to check first. Is there a better way to check for a null here? 447 if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != 9223372036854775807)) {502 if (psMetadataLookupS64(&status,row,"warp_id") && (psMetadataLookupS64(&status,row,"warp_id") != PS_MAX_S64)) { 448 503 psMetadataAddS64(where,PS_LIST_TAIL, "warpRun.warp_id", 0, "==", psMetadataLookupS64(&status,row,"warp_id")); 449 504 } 450 505 451 506 psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); 507 if (!whereClause) { 508 psError(PS_ERR_UNKNOWN, false, "Unable to generate Where Condition"); 509 return(false); 510 } 452 511 psStringAppend(&MYquery, " WHERE %s", whereClause); 453 psFree(whereClause); 454 psFree(where); 512 513 /* psTrace("debug",2,"%ld %ld %ld %ld %s\n",psMetadataLookupS64(&status,row,"exp_id"), */ 514 /* psMetadataLookupS64(&status,row,"chip_id"), */ 515 /* psMetadataLookupS64(&status,row,"cam_id"), */ 516 /* psMetadataLookupS64(&status,row,"warp_id"), */ 517 /* whereClause); */ 518 519 /* psFree(whereClause); */ 520 /* psFree(where); */ 455 521 // printf("%s\n",MYquery); 456 522 … … 465 531 return(false); 466 532 } 467 psFree(MYquery);468 533 469 534 psArray *statsoutput = p_psDBFetchResult(config->dbh); … … 478 543 } 479 544 if (!psArrayLength(statsoutput)) { 480 psWarning("dqstatstool No stats rows found ");545 psWarning("dqstatstool No stats rows found: %ld %s %s",i,MYquery,query); 481 546 psFree(colNames); 482 547 psFree(colTypes); … … 487 552 return(false); 488 553 } 554 psFree(MYquery); 555 psFree(whereClause); 556 psFree(where); 489 557 490 558 // Magic happens and the fits table is filled. … … 546 614 success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",rowItem->data.str); 547 615 } 548 616 else if (!strcmp(colType,"PS_DATA_TIME")) { 617 psString timeString = psTimeToISO(rowItem->data.V); 618 success = psMetadataAddStr(tableRow,PS_LIST_TAIL,colName,0,"",timeString); 619 psFree(timeString); 620 } 621 549 622 if (!success) { 550 psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row. ");623 psError(PS_ERR_UNKNOWN,false, "Error constructing fits table row. %s %s",colType,colName); 551 624 psFree(colNames); 552 625 psFree(colTypes); … … 562 635 } 563 636 } 564 637 // Calculate if this exposure was "good" or not. 638 bool accept = true; 639 for (long j = 0; j < decisionRules->n; j++) { 640 psMetadata *rule = decisionRules->data[j]; 641 bool status = true; 642 psString colname = psMetadataLookupStr(&status,rule,"COLNAME"); 643 if (!status) { 644 psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column name"); 645 return(false); 646 } 647 psF32 min_value = psMetadataLookupF32(&status,rule,"MINIMUM"); 648 if (!status) { 649 psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find min value for %s",colname); 650 return(false); 651 } 652 psF32 max_value = psMetadataLookupF32(&status,rule,"MAXIMUM"); 653 if (!status) { 654 psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find max value for %s",colname); 655 return(false); 656 } 657 // Not happy with this being set to a F32. Can this ever be something else? 658 psF32 value = psMetadataLookupF32(&status,tableRow,colname); 659 if (!status) { 660 psError(PS_EXIT_CONFIG_ERROR, false, "Unable to find column %s (%f %f %f) in compare with rule.",colname,min_value,max_value,value); 661 psFree(colNames); 662 psFree(colTypes); 663 psFree(contents); 664 psFree(query); 665 psFree(outTable); 666 psFree(tableRow); 667 return(false); 668 } 669 if ((value > max_value)||(value < min_value)) { 670 accept = false; 671 } 672 } 673 bool success; 674 success = psMetadataAddBool(tableRow,PS_LIST_TAIL,"ACCEPT",0,"",accept); 675 if (!success) { 676 psError(PS_ERR_UNKNOWN,false, "Error adding exposure quality to fits table row."); 677 psFree(colNames); 678 psFree(colTypes); 679 psFree(contents); 680 psFree(query); 681 psFree(outTable); 682 psFree(statsoutput); 683 psFree(statIter); 684 psFree(tableRow); 685 return(false); 686 } 687 688 // Add the row to the table array. 565 689 psArrayAdd(outTable,0,tableRow); 566 690 psFree(tableRow); … … 600 724 static bool updaterunMode(pxConfig *config) { 601 725 PS_ASSERT_PTR_NON_NULL(config,false); 602 PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", "dqstatsRun.dqstats_id", "==");603 PXOPT_LOOKUP_STR(state, config->args, "-set_state", "state", "==");604 PXOPT_LOOKUP_S16(fault, config->args, "-fault", "fault", "==");726 PXOPT_LOOKUP_S64(dqstats_id, config->args, "-dqstats_id", true, false); 727 PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); 728 PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); 605 729 606 730 if (state && ! pxIsValidState(state)) { -
trunk/ippTools/src/dqstatstoolConfig.c
r27082 r27304 49 49 psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); 50 50 psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set dqstatsRun stats", NULL); 51 psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); 51 52 52 53 // -pendingbundle -
trunk/ippconfig/recipes/dqStatsTool.config
r26369 r27304 2 2 # BUNDLE_AGE ? "2 hours"? # I'd like this to be a way to tell if we can just skip ahead, since the bundle is "old" Don't know how, though. 3 3 4 DECISION.ENGINE METADATA # Definition of the rules to determine if a given exposure is good or not. 5 RULE MULTI 6 RULE METADATA 7 COLNAME STR FWHM_MAJOR 8 MINIMUM F32 0.0 9 MAXIMUM F32 5.0 10 END 11 RULE METADATA 12 COLNAME STR FWHM_MINOR 13 MINIMUM F32 0.0 14 MAXIMUM F32 5.0 15 END 16 # TYPE VAL COLNAME MINIMUM MAXIMUM 17 # RULE VAL FWHM_MAJOR 0.0 5.0 18 # RULE VAL FWHM_MINOR 0.0 5.0 19 END 4 20 5 21 STATS.COLUMNS METADATA # Column definitions for what goes into a bundle 22 COLUMN MULTI 23 TYPE VAL TABLE ENTRY DATATYPE COLNAME 24 COLUMN VAL rawExp exp_name PS_DATA_STRING EXPNAME # Exposure name 25 COLUMN VAL rawExp dateobs PS_DATA_TIME DATEOBS # Date of observation 26 COLUMN VAL rawExp filter PS_DATA_STRING FILTER # Filter used 27 COLUMN VAL rawExp comment PS_DATA_STRING COMMENT # Obs comment 28 COLUMN VAL rawExp ra PS_DATA_F64 RA # Right ascension 29 COLUMN VAL rawExp decl PS_DATA_F64 DEC # Declination 30 COLUMN VAL rawExp exp_time PS_DATA_F32 EXPTIME # Exposure time 6 31 7 C1 METADATA 8 TABLE STR rawExp 9 ENTRY STR ra 10 DATATYPE STR PS_DATA_F64 11 COLNAME STR RA 32 COLUMN VAL camProcessedExp zpt_obs PS_DATA_F32 ZEROPOINT 33 COLUMN VAL camProcessedExp zpt_stdev PS_DATA_F32 ZEROPOINT_STDEV 34 COLUMN VAL camProcessedExp zpt_lq PS_DATA_F32 ZEROPOINT_LQ 35 COLUMN VAL camProcessedExp zpt_uq PS_DATA_F32 ZEROPOINT_UQ 36 37 COLUMN VAL camProcessedExp fwhm_major PS_DATA_F32 FWHM_MAJOR 38 COLUMN VAL camProcessedExp fwhm_major_lq PS_DATA_F32 FWHM_MAJOR_LQ 39 COLUMN VAL camProcessedExp fwhm_major_uq PS_DATA_F32 FWHM_MAJOR_UQ 40 COLUMN VAL camProcessedExp fwhm_minor PS_DATA_F32 FWHM_MINOR 41 COLUMN VAL camProcessedExp fwhm_minor_lq PS_DATA_F32 FWHM_MINOR_LQ 42 COLUMN VAL camProcessedExp fwhm_minor_uq PS_DATA_F32 FWHM_MINOR_UQ 43 44 COLUMN VAL camProcessedExp iq_fwhm_major PS_DATA_F32 IQ_FWHM_MAJOR 45 COLUMN VAL camProcessedExp iq_fwhm_major_err PS_DATA_F32 IQ_FWHM_MAJOR_ERR 46 COLUMN VAL camProcessedExp iq_fwhm_minor PS_DATA_F32 IQ_FWHM_MINOR 47 COLUMN VAL camProcessedExp iq_fwhm_minor_err PS_DATA_F32 IQ_FWHM_MINOR_ERR 48 49 COLUMN VAL camProcessedExp iq_m2 PS_DATA_F32 IQ_M2 50 COLUMN VAL camProcessedExp iq_m2_err PS_DATA_F32 IQ_M2_ERR 51 COLUMN VAL camProcessedExp iq_m2_lq PS_DATA_F32 IQ_M2_LQ 52 COLUMN VAL camProcessedExp iq_m2_uq PS_DATA_F32 IQ_M2_UQ 53 54 COLUMN VAL camProcessedExp iq_m2c PS_DATA_F32 IQ_M2C 55 COLUMN VAL camProcessedExp iq_m2c_err PS_DATA_F32 IQ_M2C_ERR 56 COLUMN VAL camProcessedExp iq_m2c_lq PS_DATA_F32 IQ_M2C_LQ 57 COLUMN VAL camProcessedExp iq_m2c_uq PS_DATA_F32 IQ_M2C_UQ 58 59 COLUMN VAL camProcessedExp iq_m2s PS_DATA_F32 IQ_M2S 60 COLUMN VAL camProcessedExp iq_m2s_err PS_DATA_F32 IQ_M2S_ERR 61 COLUMN VAL camProcessedExp iq_m2s_lq PS_DATA_F32 IQ_M2S_LQ 62 COLUMN VAL camProcessedExp iq_m2s_uq PS_DATA_F32 IQ_M2S_UQ 63 64 COLUMN VAL camProcessedExp iq_m3 PS_DATA_F32 IQ_M3 65 COLUMN VAL camProcessedExp iq_m3_err PS_DATA_F32 IQ_M3_ERR 66 COLUMN VAL camProcessedExp iq_m3_lq PS_DATA_F32 IQ_M3_LQ 67 COLUMN VAL camProcessedExp iq_m3_uq PS_DATA_F32 IQ_M3_UQ 68 69 COLUMN VAL camProcessedExp iq_m4 PS_DATA_F32 IQ_M4 70 COLUMN VAL camProcessedExp iq_m4_err PS_DATA_F32 IQ_M4_ERR 71 COLUMN VAL camProcessedExp iq_m4_lq PS_DATA_F32 IQ_M4_LQ 72 COLUMN VAL camProcessedExp iq_m4_uq PS_DATA_F32 IQ_M4_UQ 12 73 END 13 74 14 C2 METADATA15 TABLE STR rawExp16 ENTRY STR decl17 DATATYPE STR PS_DATA_F6418 COLNAME STR DEC19 END20 21 C3 METADATA22 TABLE STR camProcessedExp23 ENTRY STR iq_m224 DATATYPE STR PS_DATA_F3225 COLNAME STR IQ_M226 END27 28 C4 METADATA29 TABLE STR camProcessedExp30 ENTRY STR iq_fwhm_major31 DATATYPE STR PS_DATA_F3232 COLNAME STR IQ_FWHM_MAJOR33 END34 35 36 END37
Note:
See TracChangeset
for help on using the changeset viewer.
