IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Apr 27, 2012, 4:41:41 PM (14 years ago)
Author:
eugene
Message:

create dvomerge client for parallel operations

Location:
branches/eam_branches/ipp-20120405/Ohana/src/dvomerge
Files:
3 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/Makefile

    r31496 r33834  
    11default: all
    22help:
    3         @echo "make options: dvomerge (default)"
     3        @echo "make options: dvomerge dvomerge_client dvoconvert (default)"
    44
    55include ../../Makefile.System
    … …  
    1717FULL_LDFLAGS  = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS)
    1818
    19 dvomerge     : $(BIN)/dvomerge.$(ARCH)
    20 dvomerge     : $(BIN)/dvoconvert.$(ARCH)
     19dvomerge        : $(BIN)/dvomerge.$(ARCH)
     20dvomerge_client : $(BIN)/dvomerge_client.$(ARCH)
     21dvoconvert      : $(BIN)/dvoconvert.$(ARCH)
    2122
    22 all: dvomerge dvoconvert dvosecfilt
     23all: dvomerge dvomerge_client dvoconvert dvosecfilt
    2324
    2425#  $(SRC)/dvomergeContinue.$(ARCH).o
    … …  
    2829$(SRC)/dvomergeUpdate.$(ARCH).o \
    2930$(SRC)/dvomergeCreate.$(ARCH).o \
     31$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
    3032$(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
    3133$(SRC)/dvomergeFromList.$(ARCH).o \
    … …  
    4749
    4850$(BIN)/dvomerge.$(ARCH) : $(DVOMERGE)
     51
     52DVOMERGE_CLIENT = \
     53$(SRC)/dvomerge_client.$(ARCH).o \
     54$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
     55$(SRC)/dvomergeImageIDs.$(ARCH).o \
     56$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
     57$(SRC)/SetSignals.$(ARCH).o \
     58$(SRC)/ConfigInit.$(ARCH).o \
     59$(SRC)/Shutdown.$(ARCH).o \
     60$(SRC)/help.$(ARCH).o \
     61$(SRC)/args.$(ARCH).o \
     62$(SRC)/psps_ids.$(ARCH).o \
     63$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
     64$(SRC)/LoadCatalog.$(ARCH).o \
     65$(SRC)/build_links.$(ARCH).o \
     66$(SRC)/merge_catalogs_new.$(ARCH).o \
     67$(SRC)/merge_catalogs_old.$(ARCH).o
     68
     69$(DVOMERGE_CLIENT) : $(INC)/dvomerge.h
     70
     71$(BIN)/dvomerge_client.$(ARCH) : $(DVOMERGE_CLIENT)
    4972
    5073DVOCONVERT = \
    … …  
    99122$(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
    100123
    101 INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify
     124INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify
    102125
    103126# dependancy rules for binary code #########################

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/include/dvomerge.h

    r33657 r33834  
    1616# include <arpa/inet.h>
    1717# include <glob.h>
     18
     19# define MAX_PATH_LENGTH 1024
     20
     21int    PARALLEL;
     22int    PARALLEL_MANUAL;
     23int    PARALLEL_SERIAL;
     24
     25int    HOST_ID;
     26char  *HOSTDIR;
     27
    1828
    1929int    VERBOSE;
    … …  
    5161void       dvomerge_help          PROTO((int argc, char **argv));
    5262int        dvomerge_args          PROTO((int *argc, char **argv));
     63
     64void       dvomerge_client_usage  PROTO((void));
     65void       dvomerge_client_help   PROTO((int argc, char **argv));
     66int        dvomerge_client_args   PROTO((int *argc, char **argv));
    5367
    5468void       dvoconvert_usage       PROTO((void));
    … …  
    110124int        dvomergeFromList       PROTO((int argc, char **argv));
    111125int        dvomergeUpdate_threaded PROTO((int argc, char **argv));
     126int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType IDmap, int *secfiltMap));

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/args.c

    r33657 r33834  
    4343  }
    4444
     45  // XXX for the moment, make this selection manual.  it needs to be automatic
     46  // based on the state of the SkyTable
     47  PARALLEL = FALSE;
     48  if ((N = get_argument (*argc, argv, "-parallel"))) {
     49    PARALLEL = TRUE;
     50    remove_argument (N, argc, argv);
     51  }
     52  // this is a test mode : rather than launching the remote jobs and waiting for completion,
     53  // relphot will simply list the remote command and wait for the user to signal completion
     54  PARALLEL_MANUAL = FALSE;
     55  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
     56    PARALLEL = TRUE; // -parallel-manual implies -parallel
     57    PARALLEL_MANUAL = TRUE;
     58    remove_argument (N, argc, argv);
     59  }
     60  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
     61  // run in serial via 'system'
     62  PARALLEL_SERIAL = FALSE;
     63  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
     64    if (PARALLEL_MANUAL) {
     65      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
     66      exit (1);
     67    }
     68    PARALLEL = TRUE; // -parallel-serial implies -parallel
     69    PARALLEL_SERIAL = TRUE;
     70    remove_argument (N, argc, argv);
     71  }
     72
    4573  if ((*argc < 4) || (*argc > 6)) dvomerge_usage();
     74  return TRUE;
     75}
     76
     77/*** check for command line options ***/
     78int dvomerge_client_args (int *argc, char **argv) {
     79 
     80  int N;
     81
     82  // by definition, the client is not parallel
     83  PARALLEL = FALSE;
     84  PARALLEL_MANUAL = FALSE;
     85  PARALLEL_SERIAL = FALSE;
     86
     87  HOST_ID = 0;
     88  if ((N = get_argument (*argc, argv, "-hostID"))) {
     89    remove_argument (N, argc, argv);
     90    HOST_ID = atoi (argv[N]);
     91    remove_argument (N, argc, argv);
     92  }
     93  if (!HOST_ID) dvomerge_client_usage();
     94
     95  HOSTDIR = NULL;
     96  if ((N = get_argument (*argc, argv, "-hostdir"))) {
     97    remove_argument (N, argc, argv);
     98    HOSTDIR = strcreate (argv[N]);
     99    remove_argument (N, argc, argv);
     100  }
     101  if (!HOSTDIR) dvomerge_client_usage();
     102
     103  /* extra error messages */
     104  VERBOSE = FALSE;
     105  if ((N = get_argument (*argc, argv, "-v"))) {
     106    VERBOSE = TRUE;
     107    remove_argument (N, argc, argv);
     108  }
     109
     110  NTHREADS = 0;
     111  if ((N = get_argument (*argc, argv, "-threads"))) {
     112    remove_argument (N, argc, argv);
     113    NTHREADS = MAX(0, atoi(argv[N]));
     114    remove_argument (N, argc, argv);
     115  }
     116
     117  /*** provide additional data ***/
     118  /* restrict to a portion of the sky? */
     119  UserPatch.Rmin = 0;
     120  UserPatch.Rmax= 360;
     121  UserPatch.Dmin = -90;
     122  UserPatch.Dmax = +90;
     123  if ((N = get_argument (*argc, argv, "-region"))) {
     124    remove_argument (N, argc, argv);
     125    UserPatch.Rmin = atof (argv[N]);
     126    remove_argument (N, argc, argv);
     127    UserPatch.Rmax = atof (argv[N]);
     128    remove_argument (N, argc, argv);
     129    UserPatch.Dmin = atof (argv[N]);
     130    remove_argument (N, argc, argv);
     131    UserPatch.Dmax = atof (argv[N]);
     132    remove_argument (N, argc, argv);
     133  }
     134
     135  if ((*argc < 4) || (*argc > 6)) dvomerge_client_usage();
    46136  return TRUE;
    47137}

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomergeImageIDs.c

    r29938 r33834  
    1717  // load the image table
    1818  if (inDB.dbstate == LCK_EMPTY) {
     19    dvo_image_unlock (&inDB); // unlock input
    1920    IDmap->Nmap = 0;
    2021    return TRUE;
    … …  
    6869  // load the image table
    6970  if (inDB.dbstate == LCK_EMPTY) {
     71    dvo_image_unlock (&inDB); // unlock input
    7072    IDmap->Nmap = 0;
    7173    return TRUE;
    … …  
    8688  /* load the image table */
    8789  if (outDB.dbstate == LCK_EMPTY) {
    88     Shutdown ("only use -continue for an existing, partially merged database");
     90    Shutdown ("for dvomerge -continue or dvomerge_client, the output database must exist");
    8991  }
    9092  if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r33657 r33834  
    44
    55  int CONTINUE;
    6   off_t i, j;
    76  SkyTable *outsky, *insky;
    8   SkyList *outlist, *inlist;
    9   Catalog incatalog, outcatalog;
     7  SkyList *inlist;
    108  char filename[256], *input, *output;
    119  IDmapType IDmap;
    … …  
    107105  SetPhotcodeTable(NULL);
    108106
    109   // loop over the populated input regions
    110   for (i = 0; i < inlist[0].Nregions; i++) {
    111     if (!inlist[0].regions[i][0].table) continue;
    112     if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    113 
    114     // load / create output catalog (if catalog does not exist, it will be created)
    115     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    116     // skip empty input catalogs
    117     if (!incatalog.Naves_disk) {
    118         dvo_catalog_unlock (&incatalog);
    119         dvo_catalog_free (&incatalog);
    120         continue;
    121     }
    122 
    123     // combine only tables at equal or larger depth
    124      
    125     // load in all of the tables from input for this region
    126     // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding).  Since the regions are not infinitely small,
    127     // compare to a slightly reduced footprint
    128     float dPos = 2.0/3600.0;
    129     outlist = SkyListByBounds (outsky, inlist[0].regions[i][0].depth, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos);
    130     for (j = 0; j < outlist[0].Nregions; j++) {
    131       if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
    132 
    133       // load input catalog
    134       LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    135 
    136       dvo_update_image_IDs (&IDmap, &incatalog);
    137       merge_catalogs_old (&outsky[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap);
    138 
    139       outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    140 
    141       // if we receive a signal which would cause us to exit, wait until the full catalog is written
    142       SetProtect (TRUE);
    143       if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
    144         fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
    145         exit (1);
    146       }
    147       SetProtect (FALSE);
    148 
    149       dvo_catalog_unlock (&outcatalog);
    150       dvo_catalog_free (&outcatalog);
    151 
    152       fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name);
    153     }
    154     SkyListFree (outlist);
    155 
    156     dvo_catalog_unlock (&incatalog);
    157     dvo_catalog_free (&incatalog);
    158   }
     107  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, IDmap, secfiltMap);
    159108
    160109  // save the output sky table copy

  • branches/eam_branches/ipp-20120405/Ohana/src/dvomerge/src/help.c

    r30615 r33834  
    66  fprintf (stderr, "   OR: dvomerge (input) into (output) continue\n");
    77  fprintf (stderr, "   OR: dvomerge (input) into (output) from (list)\n");
     8  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
     9  exit (2);
     10}
     11
     12void dvomerge_client_usage (void) {
     13  fprintf (stderr, "USAGE: dvomerge_client (input) into (output)\n");
    814  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
    915  exit (2);
    … …  
    3945  fprintf (stderr, "  dvomerge (input) into (output) from (list)\n");
    4046  fprintf (stderr, "     dvomerge list implies 'continue' : list contains, eg, n1500/1688.00.cpt (one cpt per line)\n\n");
     47  fprintf (stderr, "  merge DVO databases\n");
     48  fprintf (stderr, "  optional flags:\n");
     49  fprintf (stderr, "  -v                          : verbose mode\n");
     50  fprintf (stderr, "  -help                       : this list\n");
     51  fprintf (stderr, "  -h                          : this list\n\n");
     52  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
     53  exit (2);
     54}
     55
     56void dvomerge_client_help (int argc, char **argv) {
     57
     58  /* check for help request */
     59  if (get_argument (argc, argv, "-help")) goto show_help;
     60  if (get_argument (argc, argv, "-h"))    goto show_help;
     61  return;
     62
     63show_help:
     64
     65  fprintf (stderr, "USAGE\n");
     66  fprintf (stderr, "  dvomerge_client (input) into (output)\n");
    4167  fprintf (stderr, "  merge DVO databases\n");
    4268  fprintf (stderr, "  optional flags:\n");
Note: See TracChangeset for help on using the changeset viewer.