Index: /trunk/Ohana/src/addstar/include/addstar.h =================================================================== --- /trunk/Ohana/src/addstar/include/addstar.h (revision 42711) +++ /trunk/Ohana/src/addstar/include/addstar.h (revision 42712) @@ -112,4 +112,5 @@ int ACCEPT_MOTION; // accept reference proper motion measurements int ACCEPT_TIME; // accept time stamp (or 0) +int FORCE_SINGLE_TIME; // use PHU time for all chips int NO_STARS; // ignore the stars int NO_DUPLICATE_IMAGES; // allow / skip duplicate images Index: /trunk/Ohana/src/addstar/src/LoadData.c =================================================================== --- /trunk/Ohana/src/addstar/src/LoadData.c (revision 42711) +++ /trunk/Ohana/src/addstar/src/LoadData.c (revision 42712) @@ -46,4 +46,11 @@ } images[0][Nvalid].imageID = Nvalid; + + if (FORCE_SINGLE_TIME && (i > 0)) { + if (images[0][Nvalid].tzero != images[0][0].tzero) { + fprintf (stderr, "WARNING: mismatched header times, setting all to PHU value\n"); + images[0][0].tzero = images[0][Nvalid].tzero; + } + } // XXX EAM : I seemed to have dropped the ability to support TEXT (old-style cmp format files). Index: /trunk/Ohana/src/addstar/src/args.c =================================================================== --- /trunk/Ohana/src/addstar/src/args.c (revision 42711) +++ /trunk/Ohana/src/addstar/src/args.c (revision 42712) @@ -301,4 +301,10 @@ remove_argument (N, &argc, argv); SELECT_2MASS_QUALITY = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + /* force using single time from PHU */ + FORCE_SINGLE_TIME = FALSE; + if ((N = get_argument (argc, argv, "-force-single-time"))) { + FORCE_SINGLE_TIME = TRUE; remove_argument (N, &argc, argv); } Index: /trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- /trunk/ippScripts/scripts/addstar_run.pl (revision 42711) +++ /trunk/ippScripts/scripts/addstar_run.pl (revision 42712) @@ -45,6 +45,6 @@ my ( $add_id, $camera, $stage, $stage_id, $stage_extra1, $outroot, $stageroot, $dbname, $reduction, - $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $minidvodb_host, $image_only, $verbose, - $no_update, $no_op, $redirect, $save_temps, $dvodbhost); + $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, + $no_update, $no_op, $redirect, $save_temps, $addrun_host); GetOptions( 'add_id=s' => \$add_id, # Camtool identifier @@ -67,10 +67,10 @@ 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? - 'dvodbhost|w=s' => \$dvodbhost, # miniDVO database group + 'addrun_host|w=s' => \$addrun_host, # miniDVO database host ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage --dbname --dvodbhost", + -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage --dbname --addrun_host", -exitval => 3, ) unless @@ -81,5 +81,5 @@ defined $dvodb and defined $dbname and - defined $dvodbhost and + defined $addrun_host and defined $camera; if ($stage =~ /cam/ && !defined $stage_id) { @@ -300,5 +300,5 @@ $minidvodb_path = $comp->{minidvodb_path}; $minidvodb_name = $comp->{minidvodb_name}; - $minidvodb_host = $comp->{minidvodb_host}; + $addrun_host = $comp->{addrun_host}; if (!defined($minidvodb_path)) { @@ -350,4 +350,5 @@ # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -force-single-time"; # IPP-676 time difference for chips causes dvo mosaic issue $command .= " -D CAMERA $camdir"; $command .= " -D CATDIR $minidvodb_path"; @@ -426,5 +427,5 @@ $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; -$fpaCommand .= " -minidvodb_host $dvodbhost" if defined $dvodbhost; +$fpaCommand .= " -addrun_host $addrun_host" if defined $addrun_host; $fpaCommand .= " -dbname $dbname" if defined $dbname; @@ -464,5 +465,5 @@ $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; # dont think we want it recorded (not sure) - $command .= " -minidvodb_host $dvodbhost" if defined $dvodbhost; # dont think we want it recorded (not sure) + $command .= " -addrun_host $addrun_host" if defined $addrun_host; $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- /trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 42711) +++ /trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 42712) @@ -47,20 +47,21 @@ } -my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +my ( $minidvodb, $minidvodb_id, $minidvodb_group, $minidvodb_host, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); GetOptions( - 'minidvodb|w=s' => \$minidvodb, #minidvodb database - 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id - 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id - 'camera|c=s' => \$camera, # Camera - 'dbname|d=s' => \$dbname, # Database name - 'verbose' => \$verbose, # Print to stdout - 'no-op' => \$no_op, # Don't do any operations? - 'logfile=s' => \$logfile, - 'save-temps' => \$save_temps, # Save temporary files? + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_group + 'minidvodb_host|w=s' => \$minidvodb_host, #minidvodb_host + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? ) or pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera", + -msg => "Required options: --minidvodb --minidvodb_id --minidvodb_group --minidvodb_host --camera ", -exitval => 3, ) unless @@ -68,4 +69,5 @@ defined $minidvodb_id and defined $minidvodb_group and + defined $minidvodb_host and defined $camera; Index: /trunk/ippTasks/addstar.pro =================================================================== --- /trunk/ippTasks/addstar.pro (revision 42711) +++ /trunk/ippTasks/addstar.pro (revision 42712) @@ -228,6 +228,8 @@ macro stuff - echo $LOADEXEC_ADD - echo $LOADPOLL + echo LOADEXEC_ADD: $LOADEXEC_ADD + echo LOADPOLL_ADD: $LOADPOLL_ADD + echo RUNEXEC_ADD: $RUNEXEC_ADD + echo RUNPOLL_ADD: $RUNPOLL_ADD end @@ -249,6 +251,9 @@ $addstar_revert_DB_DF = 0 -#$LOADEXEC_ADD = 300 +# loading time every N seconds $LOADEXEC_ADD = 10 +$LOADPOLL_ADD = 10 +$RUNEXEC_ADD = 5 +$RUNPOLL_ADD = 10 if ($?addstar_multiadd_limit == 0) set addstar_multiadd_limit = 0 @@ -273,17 +278,25 @@ host local - periods -poll $LOADPOLL + periods -poll $LOADPOLL_ADD periods -exec $LOADEXEC_ADD periods -timeout 300 npending 1 + date -var rundate + stdout NULL stderr $LOGDIR/addstar.exp.log task.exec - #if no stages defined we can't run + #if no stages defined we cant run if ($ADDSTAGES:n == 0) echo "what" - # if ($ADDSTAGES:n == 0) break - $run = addtool -pendingexp + if ($ADDSTAGES:n == 0) break + $run = addtool -pendingexp -addrand + + book npages addPendingExp -var addPendingExp_Npage + if ($addPendingExp_Npage > 10000) + $POLL_LIMIT = 1 + echo $addPendingExp_Npage $rundate + end #option $ADDSTAGES:$addstar_stages_DB @@ -301,5 +314,5 @@ add_poll_args run add_poll_labels run - echo $run + echo $run $rundate command $run end @@ -348,5 +361,5 @@ task.exec - #if no stages defined we can't run + #if no stages defined we cant run if ($MULTIADDSTAGES:n == 0) echo "no stages for addstar.multi, use add.addstagesmulti" # if ($MULTIADDSTAGES:n == 0) break @@ -400,13 +413,16 @@ # run the addstar script on pending exposures task addstar.exp.run - periods -poll $RUNPOLL - periods -exec $RUNEXEC + periods -poll $RUNPOLL_ADD + periods -exec $RUNEXEC_ADD periods -timeout 1200 ## we want only a single outstanding addstar job. # host local - npending 100 + npending 1000 task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC_ADD + book npages addPendingExp -var N if ($N == 0) break @@ -433,7 +449,7 @@ book getword addPendingExp $pageName image_only -var IMAGE_ONLY book getword addPendingExp $pageName dbname -var DBNAME - book getword addPendingExp $pageName dvodbhost -var DVODBHOST - - host -required $DVODBHOST + book getword addPendingExp $pageName addrun_host -var ADDRUN_HOST + + host -required $ADDRUN_HOST # specify choice of remote host based on camera and chip (class_id) @@ -480,5 +496,5 @@ stderr $LOGDIR/addstar.exp.log - $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --dvodbhost $DVODBHOST + $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --addrun_host $ADDRUN_HOST if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION @@ -501,8 +517,7 @@ end if ("$STAGE" == "fullforce_summary") - # This shouldn't need a stage_extra1, as there are no subcomponents to the ff summary. + # This shouldnt need a stage_extra1, as there are no subcomponents to the ff summary. $run = $run --stage_id $STAGE_ID end - @@ -528,4 +543,5 @@ echo command $run end + periods -exec 0.1 command $run end Index: /trunk/ippTasks/minidvodb.pro =================================================================== --- /trunk/ippTasks/minidvodb.pro (revision 42711) +++ /trunk/ippTasks/minidvodb.pro (revision 42712) @@ -7,5 +7,5 @@ -#There is a book for each task, because I don't use labels. +#There is a book for each task, because I dont use labels. if (not($?haveminidvodbBooks)) @@ -129,5 +129,5 @@ end -## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you dont want to run. macro add.minidvodb if ($0 != 8) @@ -146,4 +146,5 @@ book setword MINIDVODB_PREMERGE $1 DVODB $3 book setword MINIDVODB_PREMERGE $1 CAMERA $6 + book setword MINIDVODB_PREMERGE $1 MINIDVODB_HOST $7 book setword MINIDVODB_PREMERGE $1 STATE PENDING #active @@ -153,5 +154,5 @@ book setword MINIDVODB_ACTIVE $1 STATE PENDING - #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. + #create note that camera should be GPC1 for it to work. I couldnt figure out how to easily get this out. book newpage MINIDVODB_CREATE $1 book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 @@ -187,15 +188,13 @@ echo "minidvodb create" book listbook MINIDVODB_CREATE - echo "minidvodb_active" + echo "minidvodb active" book listbook MINIDVODB_ACTIVE end -#$LOADEXEC_MDB = 30 -$LOADEXEC_MDB = 2 -$LOADPOLL = 2 +$LOADEXEC_MDB = 60 $timeout_mdb = 2 -$LOADPOLL = 2 -$RUNEXEC = 2 +$LOADPOLL = 5 +$RUNEXEC = 30 @@ -430,6 +429,5 @@ #we only want one running at a time - host local - npending 1 + npending 100 task.exec @@ -448,4 +446,5 @@ book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName minidvodb_host -var MINIDVODB_HOST book getword minidvodbPreMergelist $pageName camera -var CAMERA book getword minidvodbPreMergelist $pageName state -var STATE @@ -455,5 +454,7 @@ #still buggy - $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + host -required $MINIDVODB_HOST + + $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID --minidvodb_host $MINIDVODB_HOST if ($DB:n == 0) Index: /trunk/ippTools/share/addtool_find_pendingexp_cam.sql =================================================================== --- /trunk/ippTools/share/addtool_find_pendingexp_cam.sql (revision 42711) +++ /trunk/ippTools/share/addtool_find_pendingexp_cam.sql (revision 42712) @@ -8,5 +8,5 @@ rawExp.telescope, rawExp.filelevel, - minidvodbRun.minidvodb_host as dvodbhost + minidvodbRun.minidvodb_host as addrun_host FROM addRun JOIN minidvodbRun Index: /trunk/ippTools/src/addtool.c =================================================================== --- /trunk/ippTools/src/addtool.c (revision 42711) +++ /trunk/ippTools/src/addtool.c (revision 42712) @@ -140,4 +140,5 @@ PXOPT_LOOKUP_BOOL(uncensored, config->args, "-uncensored", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(addrand, config->args, "-addrand", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -424,4 +425,9 @@ else if (strcmp(stage,"fullforce_summary") == 0) { psStringAppend(&query, " GROUP BY ff_id "); //needs to be checked, but should be fine: want 1 ff summary cmf. + } + + // random order of pending files + if (addrand) { + psStringAppend(&query, " order by rand()"); } @@ -1089,4 +1095,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(multiadd, config->args, "-multiadd", false); + PXOPT_LOOKUP_BOOL(addrand, config->args, "-addrand", false); psString query = NULL; @@ -1125,4 +1132,8 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } + // random order of pending files + if (addrand) { + psStringAppend(&query, " ORDER BY RAND()"); } psFree(where); Index: /trunk/ippTools/src/addtoolConfig.c =================================================================== --- /trunk/ippTools/src/addtoolConfig.c (revision 42711) +++ /trunk/ippTools/src/addtoolConfig.c (revision 42712) @@ -71,4 +71,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-addrand", 0, "set pending add_id randomly processing", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb", 0, "use minidvodb", false); @@ -127,4 +128,5 @@ psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-multiadd", 0, "for addstar multi mode, group by stage_id ", false); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-addrand", 0, "set pending add_id randomly processing", false); // -addprocessedexp