IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 22, 2012, 7:12:16 AM (14 years ago)
Author:
eugene
Message:

updating addstar -resort to handle parallel dbs

Location:
branches/eam_branches/ipp-20120405/Ohana/src/addstar
Files:
2 added
13 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/Makefile

    r33824 r33896  
    7878$(SRC)/resort_threaded.$(ARCH).o \
    7979$(SRC)/resort_unthreaded.$(ARCH).o \
     80$(SRC)/resort_unthreaded_catalogs.$(ARCH).o \
    8081$(SRC)/StarOps.$(ARCH).o \
    8182$(SRC)/ReadStarsFITS.$(ARCH).o \
    … …  
    8990$(SRC)/update_coords.$(ARCH).o \
    9091$(SRC)/psps_ids.$(ARCH).o
     92
     93# this is the client for parallel operations
     94ADDSTAR_CLIENT = \
     95$(SRC)/ConfigInit.$(ARCH).o \
     96$(SRC)/SetSignals.$(ARCH).o \
     97$(SRC)/Shutdown.$(ARCH).o \
     98$(SRC)/SkyListForStars.$(ARCH).o \
     99$(SRC)/SkyRegionUtils.$(ARCH).o \
     100$(SRC)/addstar_client.$(ARCH).o \
     101$(SRC)/args_parallel_client.$(ARCH).o \
     102$(SRC)/resort_catalog.$(ARCH).o \
     103$(SRC)/resort_threaded.$(ARCH).o \
     104$(SRC)/resort_unthreaded.$(ARCH).o \
     105$(SRC)/resort_unthreaded_catalogs.$(ARCH).o
    91106
    92107ADDSTARD = \
    … …  
    168183$(SRC)/psps_ids.$(ARCH).o
    169184
     185# this is the client for server / client addstar operations
    170186ADDSTARC = \
    171187$(SRC)/addstarc.$(ARCH).o \

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/include/addstar.h

    r33653 r33896  
    5959typedef struct sockaddr_in SockAddress;
    6060
    61 enum {M_IMAGE, M_REFLIST, M_REFCAT, M_FAKEIMAGE, M_RESORT};
     61enum {ADDSTAR_MODE_NONE, ADDSTAR_MODE_IMAGE, ADDSTAR_MODE_REFLIST, ADDSTAR_MODE_REFCAT, ADDSTAR_MODE_FAKEIMAGE, ADDSTAR_MODE_RESORT};
    6262enum {NONE, SIMPLE_CMP, SIMPLE_CMF, SIMPLE_MEF, MOSAIC_CMP, MOSAIC_CMF, MOSAIC_MEF, MOSAIC_PHU, SDSS_OBJ};
    6363/* note: MEF implies CMF */
    … …  
    132132int     OLD_RESORT;
    133133
     134int    PARALLEL;
     135int    PARALLEL_MANUAL;
     136int    PARALLEL_SERIAL;
     137int    HOST_ID;
     138char  *HOSTDIR;
     139
    134140// carries the mosaic into gstars
    135141
    … …  
    216222int        replace_match          PROTO((Average *average, Measure *measure, Stars *star));
    217223int        resort_threaded        PROTO((AddstarClientOptions *options, SkyTable *sky));
    218 int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky));
     224int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath));
     225int        resort_unthreaded_catalogs PROTO((AddstarClientOptions *options, SkyList *skylist, int hostID, char *hostpath));
    219226void       resort_catalog         PROTO((Catalog *catalog));
    220227void       resort_catalog_old     PROTO((Catalog *catalog));

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/addstar.c

    r29938 r33896  
    3434  SkyTableSetFilenames (sky, CATDIR, "cpt");
    3535 
    36   if (options.mode == M_RESORT) {
     36  if (options.mode == ADDSTAR_MODE_RESORT) {
    3737    if (NTHREADS == 0) {
    38       resort_unthreaded (&options, sky);
     38      resort_unthreaded (&options, sky, 0, NULL);
    3939    } else {
    4040      resort_threaded (&options, sky);
    … …  
    4747  /*** load in the new data (images, stars) ***/
    4848  switch (options.mode) {
    49     case M_IMAGE:
     49    case ADDSTAR_MODE_IMAGE:
    5050      stars = LoadStars (argv[1], &Nstars, &images, &Nimages, &options);
    5151
    … …  
    6262      ImageOptions (&options, images, Nimages);
    6363      break;
    64     case M_REFLIST:
     64    case ADDSTAR_MODE_REFLIST:
    6565      stars = grefstars (argv[1], options.photcode, &Nstars);
    6666      skylist = SkyListForStars (sky, -1, stars, Nstars);
    6767      break;
    68     case M_RESORT:
    69     case M_REFCAT:
     68    case ADDSTAR_MODE_RESORT:
     69    case ADDSTAR_MODE_REFCAT:
    7070      skylist = SkyListByPatch (sky, -1, &UserPatch);
    7171      break;
    72     case M_FAKEIMAGE:
     72    case ADDSTAR_MODE_FAKEIMAGE:
    7373      images = fakeimage (argv[1], &Nimages, options.photcode);
    7474      ALLOCATE (skylist, SkyList, 1);
    … …  
    137137
    138138    switch (options.mode) {
    139       case M_IMAGE:
     139      case ADDSTAR_MODE_IMAGE:
    140140        Nsubset = Nstars;
    141141        if (options.closest) {
    … …  
    145145        }
    146146        break;
    147       case M_REFCAT:
     147      case ADDSTAR_MODE_REFCAT:
    148148        stars = greference (argv[1], skylist[0].regions[i], options.photcode, &Nstars);
    149       case M_REFLIST:
     149      case ADDSTAR_MODE_REFLIST:
    150150        subset = find_subset (skylist[0].regions[i], stars, Nstars, &Nsubset);
    151151        if (options.closest) {
    … …  
    156156        if (Nsubset) free (subset);
    157157        break;
    158       case M_RESORT:
     158      case ADDSTAR_MODE_RESORT:
    159159        if (options.nosort == 3) catalog.sorted = FALSE;
    160160
    … …  
    192192    dvo_catalog_free (&catalog);
    193193
    194     if (options.mode == M_REFCAT) free (stars);
     194    if (options.mode == ADDSTAR_MODE_REFCAT) free (stars);
    195195  }
    196196
    … …  
    216216
    217217  /* add the new images and save */
    218   if (options.mode == M_IMAGE) {
     218  if (options.mode == ADDSTAR_MODE_IMAGE) {
    219219    dvo_image_addrows (&db, images, Nimages);
    220220    SetProtect (TRUE);

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/addstarc.c

    r21153 r33896  
    2020  /* send new data to server */
    2121  switch (options.mode) {
    22     case M_IMAGE:
     22    case ADDSTAR_MODE_IMAGE:
    2323      /* load data */
    2424      stars = LoadStars (argv[1], &Nstars, &images, &Nimages, &options);
    … …  
    4040      break;
    4141
    42     case M_REFLIST:
     42    case ADDSTAR_MODE_REFLIST:
    4343      /* load data */
    4444      stars = grefstars (argv[1], options.photcode, &Nstars);
    … …  
    5050      break;
    5151
    52     case M_REFCAT:
     52    case ADDSTAR_MODE_REFCAT:
    5353      /* send data to server */
    5454      SendCommand (BindSocket, 5, "REFCT");

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/addstart.c

    r25757 r33896  
    3131
    3232    switch (dataset[0].options[0].mode) {
    33       case M_IMAGE:
     33      case ADDSTAR_MODE_IMAGE:
    3434        UpdateDatabase_Image (dataset[0].options, dataset[0].images, dataset[0].Nimages, dataset[0].mosaic, dataset[0].stars, dataset[0].Nstars);
    3535        continue;
    3636
    37       case M_REFLIST:
     37      case ADDSTAR_MODE_REFLIST:
    3838        UpdateDatabase_Reflist (dataset[0].options, dataset[0].stars, dataset[0].Nstars);
    3939        continue;
    4040
    41       case M_REFCAT:
     41      case ADDSTAR_MODE_REFCAT:
    4242        UpdateDatabase_Refcat (dataset[0].options, dataset[0].patch, dataset[0].refcat);
    4343        continue;

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args.c

    r31669 r33896  
    2121
    2222  /* basic mode: image, list, refcat */
    23   options.mode = M_IMAGE;
     23  options.mode = ADDSTAR_MODE_IMAGE;
    2424  if ((N = get_argument (argc, argv, "-ref"))) {
    25     options.mode = M_REFLIST;
     25    options.mode = ADDSTAR_MODE_REFLIST;
    2626    remove_argument (N, &argc, argv);
    2727  }
    2828  if ((N = get_argument (argc, argv, "-cat"))) {
    29     options.mode = M_REFCAT;
     29    options.mode = ADDSTAR_MODE_REFCAT;
    3030    remove_argument (N, &argc, argv);
    3131  }
    3232  if ((N = get_argument (argc, argv, "-resort"))) {
    33     options.mode = M_RESORT;
     33    options.mode = ADDSTAR_MODE_RESORT;
    3434    remove_argument (N, &argc, argv);
    3535  }
    3636
    3737  if ((N = get_argument (argc, argv, "-fakeimage"))) {
    38     options.mode = M_FAKEIMAGE;
     38    options.mode = ADDSTAR_MODE_FAKEIMAGE;
    3939    remove_argument (N, &argc, argv);
    4040    FAKE_RA = atof (argv[N]);
    … …  
    9898    remove_argument (N, &argc, argv);
    9999  } else {
    100       if (options.mode == M_IMAGE) goto allow;
    101       if (options.mode == M_FAKEIMAGE) goto allow;
    102       if (options.mode == M_REFLIST) goto allow;
    103       if (options.mode == M_REFCAT) {
     100      if (options.mode == ADDSTAR_MODE_IMAGE) goto allow;
     101      if (options.mode == ADDSTAR_MODE_FAKEIMAGE) goto allow;
     102      if (options.mode == ADDSTAR_MODE_REFLIST) goto allow;
     103      if (options.mode == ADDSTAR_MODE_REFCAT) {
    104104          fprintf (stderr, "you have requested uploading from a catalog to the entire sky in one pass\n");
    105105      }
    106       if (options.mode == M_RESORT) {
     106      if (options.mode == ADDSTAR_MODE_RESORT) {
    107107          fprintf (stderr, "you have requested resorting the entire sky in one pass\n");
    108108      }
    … …  
    330330  }
    331331
    332   if ((options.mode == M_RESORT) && (argc == 1)) return (options);
     332  // XXX for the moment, make this selection manual.  it needs to be automatic
     333  // based on the state of the SkyTable
     334  PARALLEL = FALSE;
     335  if ((N = get_argument (argc, argv, "-parallel"))) {
     336    PARALLEL = TRUE;
     337    remove_argument (N, &argc, argv);
     338  }
     339  // this is a test mode : rather than launching the remote jobs and waiting for completion,
     340  // relphot will simply list the remote command and wait for the user to signal completion
     341  PARALLEL_MANUAL = FALSE;
     342  if ((N = get_argument (argc, argv, "-parallel-manual"))) {
     343    PARALLEL = TRUE; // -parallel-manual implies -parallel
     344    PARALLEL_MANUAL = TRUE;
     345    remove_argument (N, &argc, argv);
     346  }
     347  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
     348  // run in serial via 'system'
     349  PARALLEL_SERIAL = FALSE;
     350  if ((N = get_argument (argc, argv, "-parallel-serial"))) {
     351    if (PARALLEL_MANUAL) {
     352      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
     353      exit (1);
     354    }
     355    PARALLEL = TRUE; // -parallel-serial implies -parallel
     356    PARALLEL_SERIAL = TRUE;
     357    remove_argument (N, &argc, argv);
     358  }
     359  if (PARALLEL) {
     360    if (options.mode != ADDSTAR_MODE_RESORT) {
     361      fprintf (stderr, "parallel mode is only valid for -resort mode\n");
     362      exit (2);
     363    }
     364  }
     365
     366  if ((options.mode == ADDSTAR_MODE_RESORT) && (argc == 1)) return (options);
    333367  if (argc == 2) return (options);
    334368
    335   if ((options.mode == M_REFLIST) && (options.photcode == 0)) {
     369  if ((options.mode == ADDSTAR_MODE_REFLIST) && (options.photcode == 0)) {
    336370    fprintf (stderr, "photcode must be specified for -ref\n");
    337371    exit (2);

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args_client.c

    r18374 r33896  
    1818
    1919  /* basic mode: image, list, refcat */
    20   options.mode = M_IMAGE;
     20  options.mode = ADDSTAR_MODE_IMAGE;
    2121  if ((N = get_argument (argc, argv, "-ref"))) {
    22     options.mode = M_REFLIST;
     22    options.mode = ADDSTAR_MODE_REFLIST;
    2323    remove_argument (N, &argc, argv);
    2424  }
    2525  if ((N = get_argument (argc, argv, "-cat"))) {
    26     options.mode = M_REFCAT;
     26    options.mode = ADDSTAR_MODE_REFCAT;
    2727    remove_argument (N, &argc, argv);
    2828  }

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args_load2mass.c

    r24977 r33896  
    1818
    1919  /* basic mode: image, list, refcat */
    20   options.mode = M_REFCAT;
     20  options.mode = ADDSTAR_MODE_REFCAT;
    2121
    2222  /*** provide additional data ***/

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args_loadsupercos.c

    r33653 r33896  
    1818
    1919  /* basic mode: image, list, refcat */
    20   options.mode = M_REFCAT;
     20  options.mode = ADDSTAR_MODE_REFCAT;
    2121
    2222  /* we do not allow a subset to be extracted -- all or nothing, babe */

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args_loadwise.c

    r33824 r33896  
    1919
    2020  /* basic mode: image, list, refcat */
    21   options.mode = M_REFCAT;
     21  options.mode = ADDSTAR_MODE_REFCAT;
    2222
    2323  /* we do not allow a subset to be extracted -- all or nothing, babe */

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/args_sedstar.c

    r18374 r33896  
    1818
    1919  /* basic mode: image, list, refcat */
    20   options.mode = M_REFCAT;
     20  options.mode = ADDSTAR_MODE_REFCAT;
    2121
    2222  /*** provide additional data ***/

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded.c

    r33895 r33896  
    11# include "addstar.h"
    22
    3 /* upgrade to parallel mode:
    4    
    5    setup up skylist and call resort_unthreaded_catalogs ()
    6    
    7    resort_unthreaded_catalogs has a parallel and serial version, etc.
    8 
    9  */
    10 
    11 int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky) {
    12 
    13   off_t i;
    14   off_t Naverage, Nmeasure;
    15   Catalog catalog;
     3int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath) {
    164
    175  double dtime;
    … …  
    5240  }
    5341
    54   resort_unthreaded_catalogs ()
     42  resort_unthreaded_catalogs (options, 0, NULL)
    5543
    5644  gettimeofday (&stop, NULL);

  • branches/eam_branches/ipp-20120405/Ohana/src/addstar/src/resort_unthreaded_catalogs.c

    r33895 r33896  
    11# include "addstar.h"
    22
    3 int resort_unthreaded_catalogs (int hostID, char *hostpath) {
     3// pass in options and skylist?
     4int resort_unthreaded_catalogs (AddstarClientOptions *options, SkyList *skylist, int hostID, char *hostpath) {
     5
     6  off_t i;
     7  off_t Naverage, Nmeasure;
     8  Catalog catalog;
    49
    510  if (PARALLEL && !hostID) {
    6     resort_unthreaded_parallel ();
     11    resort_unthreaded_parallel (options, skylist);
    712    return TRUE;
    813  }
    … …  
    1520    if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue;
    1621
    17     // XXX BLOCK here for an empty thread
    18 
    19     // set the parameters which guide catalog open/load/create
    20     catalog.filename  = skylist[0].filename[i];
    21 
     22    // chose the catalog file (local or remote?)
    2223    char hostfile[1024];
    2324    snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name);
    2425    catalog.filename  = hostID ? hostfile : skylist[0].filename[i];
    2526
     27    // set the parameters which guide catalog open/load/create
    2628    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    2729    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
    … …  
    4547    if (options->nosort == 3) catalog.sorted = FALSE;
    4648
    47     // XXX send these to a free thread
    48 
    4949    if (OLD_RESORT) {
    5050      resort_catalog_old (&catalog);
    … …  
    5353    }
    5454
    55     // XXX wait for completed threads
    56 
    57     /* report total updated values */
     55    // report total updated values
    5856    Naverage += catalog.Naverage;
    5957    Nmeasure += catalog.Nmeasure;
    … …  
    6664    dvo_catalog_free (&catalog);
    6765  }
     66  return TRUE;
    6867}
    6968
    7069// CATDIR is supplied globally
    7170# define DEBUG 1
    72 int resort_unthreaded_parallel (SkyList *sky) {
    73 
    74   // launch the setphot_client jobs to the parallel hosts
     71int resort_unthreaded_parallel (AddstarClientOptions *options, SkyList *sky) {
    7572
    7673  // load the list of hosts
    … …  
    8986    table->hosts[i].pathname = tmppath;
    9087
    91     // options / arguments that can affect relphot_client -update-objects:
    92     // VERBOSE, VERBOSE2
    93     // KEEP_UBERCAL
    94     // RESET (-reset)
    95     // TimeSelect -time
    96     // DophotSelect
    97     // (note that psfQual is applied rigidly at 0.85, as is the galaxy test)
    98     // MAG_LIM
    99     // SIGMA_LIM
    100     // ImagSelect, ImagMin, ImagMax
    101     // MaxDensityUse, MaxDensityValue
    102 
    10388    char command[1024];
    104     snprintf (command, 1024, "relphot_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -D CAMERA %s -D STAR_TOOFEW %d -minerror %f",
    105               table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, CAMERA, STAR_TOOFEW, MIN_ERROR);
     89    snprintf (command, 1024, "addstar_client -resort -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f",
     90              table->hosts[i].hostID, CATDIR, table->hosts[i].pathname,
     91              UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
    10692
    10793    char tmpline[1024];
    10894    if (VERBOSE)       { snprintf (tmpline, 1024, "%s -v",              command);                    strcpy (command, tmpline); }
    10995    if (VERBOSE2)      { snprintf (tmpline, 1024, "%s -vv",             command);                    strcpy (command, tmpline); }
    110     if (RESET)         { snprintf (tmpline, 1024, "%s -reset",          command);                    strcpy (command, tmpline); }
    111     if (UPDATE)        { snprintf (tmpline, 1024, "%s -update",         command);                    strcpy (command, tmpline); }
    112     if (!KEEP_UBERCAL) { snprintf (tmpline, 1024, "%s -reset-ubercal",  command);                    strcpy (command, tmpline); }
    11396
    11497    fprintf (stderr, "command: %s\n", command);
    … …  
    119102      int status = system (command);
    120103      if (status) {
    121         fprintf (stderr, "ERROR running relphot_client\n");
     104        fprintf (stderr, "ERROR running addstar_client\n");
    122105        exit (2);
    123106      }
    … …  
    135118
    136119  if (PARALLEL_MANUAL) {
    137     fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
     120    fprintf (stderr, "run the addstar_client commands above.  when these are done, hit return\n");
    138121    getchar();
    139122  }
Note: See TracChangeset for help on using the changeset viewer.