IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42712


Ignore:
Timestamp:
Jul 18, 2024, 2:55:58 PM (2 years ago)
Author:
cclin33
Message:

fix mis-match mosaic issue when running addstar, optimize addstar program

Location:
trunk
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/addstar/include/addstar.h

    r41475 r42712  
    112112int     ACCEPT_MOTION;  // accept reference proper motion measurements
    113113int     ACCEPT_TIME;    // accept time stamp (or 0)
     114int     FORCE_SINGLE_TIME;    // use PHU time for all chips
    114115int     NO_STARS;       // ignore the stars
    115116int     NO_DUPLICATE_IMAGES; // allow / skip duplicate images

  • trunk/Ohana/src/addstar/src/LoadData.c

    r39273 r42712  
    4646    }
    4747    images[0][Nvalid].imageID = Nvalid;
     48
     49    if (FORCE_SINGLE_TIME && (i > 0)) {
     50      if (images[0][Nvalid].tzero != images[0][0].tzero) {
     51        fprintf (stderr, "WARNING: mismatched header times, setting all to PHU value\n");
     52        images[0][0].tzero = images[0][Nvalid].tzero;
     53      }
     54    }
    4855
    4956    // XXX EAM : I seemed to have dropped the ability to support TEXT (old-style cmp format files).

  • trunk/Ohana/src/addstar/src/args.c

    r41475 r42712  
    301301    remove_argument (N, &argc, argv);
    302302    SELECT_2MASS_QUALITY = strcreate (argv[N]);
     303    remove_argument (N, &argc, argv);
     304  }
     305  /* force using single time from PHU */
     306  FORCE_SINGLE_TIME = FALSE;
     307  if ((N = get_argument (argc, argv, "-force-single-time"))) {
     308    FORCE_SINGLE_TIME = TRUE;
    303309    remove_argument (N, &argc, argv);
    304310  }

  • trunk/ippScripts/scripts/addstar_run.pl

    r41213 r42712  
    4545
    4646my ( $add_id, $camera, $stage, $stage_id, $stage_extra1, $outroot, $stageroot, $dbname, $reduction,
    47      $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $minidvodb_host, $image_only, $verbose,
    48      $no_update, $no_op, $redirect, $save_temps, $dvodbhost);
     47     $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose,
     48     $no_update, $no_op, $redirect, $save_temps, $addrun_host);
    4949GetOptions(
    5050    'add_id=s'          => \$add_id, # Camtool identifier
    … …  
    6767    'redirect-output'   => \$redirect,
    6868    'save-temps'        => \$save_temps, # Save temporary files?
    69     'dvodbhost|w=s' => \$dvodbhost, # miniDVO database group
     69    'addrun_host|w=s' => \$addrun_host, # miniDVO database host
    7070    ) or pod2usage( 2 );
    7171
    7272pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    7373pod2usage(
    74           -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage --dbname --dvodbhost",
     74          -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage --dbname --addrun_host",
    7575          -exitval => 3,
    7676          ) unless
    … …  
    8181    defined $dvodb and
    8282    defined $dbname and
    83     defined $dvodbhost and
     83    defined $addrun_host and
    8484    defined $camera;
    8585if ($stage =~ /cam/ && !defined $stage_id) {
    … …  
    300300        $minidvodb_path = $comp->{minidvodb_path};
    301301        $minidvodb_name = $comp->{minidvodb_name};
    302         $minidvodb_host = $comp->{minidvodb_host};
     302        $addrun_host = $comp->{addrun_host};
    303303
    304304        if (!defined($minidvodb_path)) {
    … …  
    350350                # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
    351351                $command  = "$addstar -update"; # XXX optionally set -update?
     352                $command .= " -force-single-time"; # IPP-676 time difference for chips causes dvo mosaic issue
    352353                $command .= " -D CAMERA $camdir";
    353354                $command .= " -D CATDIR $minidvodb_path";
    … …  
    426427$fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
    427428$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
    428 $fpaCommand .= " -minidvodb_host $dvodbhost" if defined $dvodbhost;
     429$fpaCommand .= " -addrun_host $addrun_host" if defined $addrun_host;
    429430$fpaCommand .= " -dbname $dbname" if defined $dbname;
    430431
    … …  
    464465        $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
    465466        $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; # dont think we want it recorded (not sure)
    466         $command .= " -minidvodb_host $dvodbhost" if defined $dvodbhost; # dont think we want it recorded (not sure)
     467        $command .= " -addrun_host $addrun_host" if defined $addrun_host;
    467468        $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1;
    468469        $command .= " -dbname $dbname" if defined $dbname;

  • trunk/ippScripts/scripts/minidvodb_premerge.pl

    r39491 r42712  
    4747}
    4848
    49 my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps);
     49my ( $minidvodb, $minidvodb_id, $minidvodb_group, $minidvodb_host, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps);
    5050GetOptions(
    51     'minidvodb|w=s'     => \$minidvodb, #minidvodb database
    52     'minidvodb_id|w=s'  => \$minidvodb_id, #minidvodb_id
    53     'minidvodb_group|w=s'  => \$minidvodb_group, #minidvodb_id
    54     'camera|c=s'        => \$camera, # Camera
    55     'dbname|d=s'        => \$dbname, # Database name
    56     'verbose'           => \$verbose,   # Print to stdout
    57     'no-op'             => \$no_op, # Don't do any operations?
    58     'logfile=s'         => \$logfile,
    59     'save-temps'        => \$save_temps, # Save temporary files?
     51    'minidvodb|w=s'        => \$minidvodb, #minidvodb database
     52    'minidvodb_id|w=s'     => \$minidvodb_id, #minidvodb_id
     53    'minidvodb_group|w=s'  => \$minidvodb_group, #minidvodb_group
     54    'minidvodb_host|w=s'   => \$minidvodb_host, #minidvodb_host
     55    'camera|c=s'           => \$camera, # Camera
     56    'dbname|d=s'           => \$dbname, # Database name
     57    'verbose'              => \$verbose,   # Print to stdout
     58    'no-op'                => \$no_op, # Don't do any operations?
     59    'logfile=s'            => \$logfile,
     60    'save-temps'           => \$save_temps, # Save temporary files?
    6061    ) or pod2usage( 2 );
    6162
    6263pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    6364pod2usage(
    64           -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera",
     65          -msg => "Required options: --minidvodb --minidvodb_id --minidvodb_group --minidvodb_host --camera ",
    6566          -exitval => 3,
    6667          ) unless
    … …  
    6869    defined $minidvodb_id and
    6970    defined $minidvodb_group and
     71    defined $minidvodb_host and
    7072    defined $camera;
    7173

  • trunk/ippTasks/addstar.pro

    r41213 r42712  
    228228
    229229macro stuff
    230    echo $LOADEXEC_ADD
    231    echo $LOADPOLL
     230   echo LOADEXEC_ADD: $LOADEXEC_ADD
     231   echo LOADPOLL_ADD: $LOADPOLL_ADD
     232   echo RUNEXEC_ADD:  $RUNEXEC_ADD
     233   echo RUNPOLL_ADD:  $RUNPOLL_ADD
    232234end
    233235
    … …  
    249251$addstar_revert_DB_DF = 0
    250252
    251 #$LOADEXEC_ADD = 300
     253# loading time every N seconds
    252254$LOADEXEC_ADD = 10
     255$LOADPOLL_ADD = 10
     256$RUNEXEC_ADD = 5
     257$RUNPOLL_ADD = 10
    253258
    254259if ($?addstar_multiadd_limit == 0) set addstar_multiadd_limit = 0
    … …  
    273278  host         local
    274279
    275   periods      -poll $LOADPOLL
     280  periods      -poll $LOADPOLL_ADD
    276281  periods      -exec $LOADEXEC_ADD
    277282  periods      -timeout 300
    278283  npending     1
    279284
     285  date -var rundate
     286
    280287  stdout NULL
    281288  stderr $LOGDIR/addstar.exp.log
    282289
    283290  task.exec
    284     #if no stages defined we can't run
     291    #if no stages defined we cant run
    285292    if ($ADDSTAGES:n == 0) echo "what"
    286  #   if ($ADDSTAGES:n == 0) break
    287     $run = addtool -pendingexp
     293    if ($ADDSTAGES:n == 0) break
     294    $run = addtool -pendingexp -addrand
     295
     296    book npages addPendingExp  -var addPendingExp_Npage
     297    if ($addPendingExp_Npage > 10000)
     298      $POLL_LIMIT = 1
     299      echo $addPendingExp_Npage $rundate
     300    end
    288301   
    289302    #option $ADDSTAGES:$addstar_stages_DB
    … …  
    301314    add_poll_args run
    302315    add_poll_labels run
    303     echo $run
     316    echo $run $rundate
    304317    command $run
    305318  end
    … …  
    348361
    349362  task.exec
    350     #if no stages defined we can't run
     363    #if no stages defined we cant run
    351364    if ($MULTIADDSTAGES:n == 0) echo "no stages for addstar.multi, use add.addstagesmulti"
    352365 #   if ($MULTIADDSTAGES:n == 0) break
    … …  
    400413# run the addstar script on pending exposures
    401414task           addstar.exp.run
    402   periods      -poll $RUNPOLL
    403   periods      -exec $RUNEXEC
     415  periods      -poll $RUNPOLL_ADD
     416  periods      -exec $RUNEXEC_ADD
    404417  periods      -timeout 1200
    405418
    406419  ## we want only a single outstanding addstar job. 
    407420#  host         local
    408   npending     100
     421  npending     1000
    409422
    410423  task.exec
     424    # if we are unable to run the 'exec', use a long retry time
     425    periods -exec $RUNEXEC_ADD
     426
    411427    book npages addPendingExp -var N
    412428    if ($N == 0) break
    … …  
    433449    book getword addPendingExp $pageName image_only -var IMAGE_ONLY
    434450    book getword addPendingExp $pageName dbname -var DBNAME
    435     book getword addPendingExp $pageName dvodbhost  -var DVODBHOST
    436 
    437     host -required $DVODBHOST
     451    book getword addPendingExp $pageName addrun_host  -var ADDRUN_HOST
     452
     453    host -required $ADDRUN_HOST
    438454
    439455    # specify choice of remote host based on camera and chip (class_id)
    … …  
    480496    stderr $LOGDIR/addstar.exp.log
    481497
    482     $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --dvodbhost $DVODBHOST
     498    $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output --addrun_host $ADDRUN_HOST
    483499    if ("$REDUCTION" != "NULL")
    484500      $run = $run --reduction $REDUCTION
    … …  
    501517    end
    502518    if ("$STAGE" == "fullforce_summary")
    503     # This shouldn't need a stage_extra1, as there are no subcomponents to the ff summary.
     519    # This shouldnt need a stage_extra1, as there are no subcomponents to the ff summary.
    504520      $run = $run --stage_id $STAGE_ID
    505521    end
    506 
    507522
    508523
    … …  
    528543      echo command $run
    529544    end
     545    periods -exec 0.1
    530546    command $run
    531547  end

  • trunk/ippTasks/minidvodb.pro

    r41215 r42712  
    77
    88
    9 #There is a book for each task, because I don't use labels.
     9#There is a book for each task, because I dont use labels.
    1010
    1111if (not($?haveminidvodbBooks))
    … …  
    129129end
    130130
    131 ## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run.
     131## you get no choice - you add all of them in at the same time. you can always turn off the tasks you dont want to run.
    132132macro add.minidvodb
    133133  if ($0 != 8)
    … …  
    146146  book setword MINIDVODB_PREMERGE $1 DVODB $3
    147147  book setword MINIDVODB_PREMERGE $1 CAMERA $6
     148  book setword MINIDVODB_PREMERGE $1 MINIDVODB_HOST $7
    148149  book setword MINIDVODB_PREMERGE $1 STATE PENDING
    149150  #active 
    … …  
    153154  book setword MINIDVODB_ACTIVE $1 STATE PENDING
    154155
    155   #create  note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out.
     156  #create  note that camera should be GPC1 for it to work. I couldnt figure out how to easily get this out.
    156157  book newpage MINIDVODB_CREATE $1
    157158  book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1
    … …  
    187188  echo "minidvodb create"
    188189  book listbook MINIDVODB_CREATE
    189   echo "minidvodb_active"
     190  echo "minidvodb active"
    190191  book listbook MINIDVODB_ACTIVE
    191192end
    192193
    193194
    194 #$LOADEXEC_MDB = 30
    195 $LOADEXEC_MDB = 2
    196 $LOADPOLL = 2
     195$LOADEXEC_MDB = 60
    197196$timeout_mdb  = 2
    198 $LOADPOLL = 2
    199 $RUNEXEC = 2
     197$LOADPOLL = 5
     198$RUNEXEC = 30
    200199
    201200
    … …  
    430429  #we only want one running at a time
    431430
    432   host         local
    433   npending     1
     431  npending     100
    434432
    435433  task.exec
    … …  
    448446    book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
    449447    book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH
     448    book getword minidvodbPreMergelist $pageName minidvodb_host -var MINIDVODB_HOST
    450449    book getword minidvodbPreMergelist $pageName camera -var CAMERA
    451450    book getword minidvodbPreMergelist $pageName state -var STATE
    … …  
    455454    #still buggy
    456455
    457     $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
     456    host -required $MINIDVODB_HOST
     457
     458    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID --minidvodb_host $MINIDVODB_HOST
    458459   
    459460  if ($DB:n == 0)

  • trunk/ippTools/share/addtool_find_pendingexp_cam.sql

    r41213 r42712  
    88    rawExp.telescope,
    99    rawExp.filelevel,
    10     minidvodbRun.minidvodb_host as dvodbhost
     10    minidvodbRun.minidvodb_host as addrun_host
    1111FROM addRun
    1212JOIN minidvodbRun

  • trunk/ippTools/src/addtool.c

    r41213 r42712  
    140140    PXOPT_LOOKUP_BOOL(uncensored, config->args, "-uncensored", false);
    141141    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     142    PXOPT_LOOKUP_BOOL(addrand, config->args,  "-addrand", false);
    142143
    143144    PXOPT_LOOKUP_BOOL(pretend,    config->args, "-pretend", false);
    … …  
    424425    else if (strcmp(stage,"fullforce_summary") == 0) {
    425426      psStringAppend(&query, " GROUP BY ff_id ");  //needs to be checked, but should be fine: want 1 ff summary cmf.
     427    }
     428
     429    // random order of pending files
     430    if (addrand) {
     431      psStringAppend(&query, " order by rand()");
    426432    }
    427433
    … …  
    10891095    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    10901096    PXOPT_LOOKUP_BOOL(multiadd, config->args, "-multiadd", false);
     1097    PXOPT_LOOKUP_BOOL(addrand, config->args,  "-addrand", false);
    10911098   
    10921099    psString query = NULL;
    … …  
    11251132        psStringAppend(&query, " AND %s", whereClause);
    11261133        psFree(whereClause);
     1134    }
     1135    // random order of pending files
     1136    if (addrand) {
     1137      psStringAppend(&query, " ORDER BY RAND()");
    11271138    }
    11281139    psFree(where);

  • trunk/ippTools/src/addtoolConfig.c

    r41654 r42712  
    7171    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",            0, "use the simple output format", false);
    7272    psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit",              0, "limit result set to N items", 0);
     73    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-addrand",           0, "set pending add_id randomly processing", false);
    7374
    7475    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb",            0, "use minidvodb", false);
    … …  
    127128    psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple",           0, "use the simple output format", false);
    128129    psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-multiadd",           0, "for addstar multi mode, group by stage_id ", false);
     130    psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-addrand",           0, "set pending add_id randomly processing", false);
    129131
    130132    // -addprocessedexp
Note: See TracChangeset for help on using the changeset viewer.