IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 37089


Ignore:
Timestamp:
Jul 20, 2014, 9:40:20 AM (12 years ago)
Author:
eugene
Message:

resort should now work for both threaded and unthreaded cases, in both parallel and non-parallel context, and with the new or old resort method

Location:
branches/eam_branches/ipp-20140717/Ohana/src/addstar
Files:
1 added
1 deleted
9 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/Makefile

    r35263 r37089  
    7878$(SRC)/parse_time.$(ARCH).o \
    7979$(SRC)/replace_match.$(ARCH).o \
    80 $(SRC)/resort_catalog.$(ARCH).o \
     80$(SRC)/resort_catalogs.$(ARCH).o \
    8181$(SRC)/resort_threaded.$(ARCH).o \
    8282$(SRC)/resort_unthreaded.$(ARCH).o \
    83 $(SRC)/resort_unthreaded_catalogs.$(ARCH).o \
     83$(SRC)/resort_catalog.$(ARCH).o \
    8484$(SRC)/StarOps.$(ARCH).o \
    8585$(SRC)/ReadStarsFITS.$(ARCH).o \
    … …  
    105105$(SRC)/addstar_client.$(ARCH).o \
    106106$(SRC)/args_parallel_client.$(ARCH).o \
    107 $(SRC)/resort_catalog.$(ARCH).o \
     107$(SRC)/resort_catalogs.$(ARCH).o \
    108108$(SRC)/resort_threaded.$(ARCH).o \
    109109$(SRC)/resort_unthreaded.$(ARCH).o \
    110 $(SRC)/resort_unthreaded_catalogs.$(ARCH).o
     110$(SRC)/resort_catalog.$(ARCH).o
    111111
    112112ADDSTARD = \

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/include/addstar.h

    r37036 r37089  
    237237Stars     *rd_gsc                 PROTO((char *filename, unsigned int *nstars));
    238238int        replace_match          PROTO((Average *average, Measure *measure, Stars *star));
    239 int        resort_threaded        PROTO((AddstarClientOptions *options, SkyTable *sky));
    240 int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath));
    241 int        resort_unthreaded_catalogs PROTO((AddstarClientOptions *options, SkyList *skylist, int hostID, char *hostpath));
     239int        resort_catalogs        PROTO((AddstarClientOptions *options, SkyTable *sky));
     240int        resort_catalogs_parallel PROTO((AddstarClientOptions *options, SkyList *sky));
     241int        resort_threaded        PROTO((SkyList *skylist, int ForceSort));
     242int        resort_unthreaded      PROTO((SkyList *skylist, int ForceSort));
    242243void       resort_catalog         PROTO((Catalog *catalog));
    243244void       resort_catalog_old     PROTO((Catalog *catalog));
     245
    244246Stars     *ReadStarsFITS          PROTO((FILE *f, Header *header, Header *in_theader, unsigned int *nstars));
    245247Stars     *ReadStarsTEXT          PROTO((FILE *f, unsigned int *nstars));
    … …  
    316318void *memrchr(const void *s, int c, size_t n);
    317319int addstar_create_ID ();
     320int strextend (char *input, char *format,...);
    318321
    319322/**

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/addstar.c

    r37036 r37089  
    4040 
    4141  if (options.mode == ADDSTAR_MODE_RESORT) {
    42     if (NTHREADS == 0) {
    43       resort_unthreaded (&options, sky, 0, NULL);
    44     } else {
    45       resort_threaded (&options, sky);
    46     }
     42    resort_catalogs (&options, sky);
    4743    exit (0);
    4844  }
    … …  
    7167      skylist = SkyListForStars (sky, -1, stars, Nstars);
    7268      break;
    73     case ADDSTAR_MODE_RESORT:
    7469    case ADDSTAR_MODE_REFCAT:
    7570      skylist = SkyListByPatch (sky, -1, &UserPatch);
    … …  
    161156        if (Nsubset) free (subset);
    162157        break;
    163       case ADDSTAR_MODE_RESORT:
    164         if (options.nosort == 3) catalog.sorted = FALSE;
    165 
    166         // no need to resort empty catalogs
    167         if (catalog.Naves_disk == 0) {
    168           dvo_catalog_unlock (&catalog);
    169           dvo_catalog_free (&catalog);
    170           continue;
    171         }
    172 
    173         if (OLD_RESORT) {
    174           resort_catalog_old (&catalog);
    175         } else {
    176           resort_catalog (&catalog);
    177         }
    178         Nsubset = 1;
    179         break;
    180158    }
    181159    /* report total updated values */

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/addstar_client.c

    r33963 r37089  
    2525
    2626  if (options.mode == ADDSTAR_MODE_RESORT) {
    27     resort_unthreaded (&options, sky, HOST_ID, HOSTDIR);
     27    resort_catalogs (&options, sky);
    2828    exit (0);
    2929  }

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/args.c

    r35760 r37089  
    3434    remove_argument (N, &argc, argv);
    3535  }
     36  OLD_RESORT = FALSE;
     37  if ((N = get_argument (argc, argv, "-old-resort"))) {
     38    remove_argument (N, &argc, argv);
     39    OLD_RESORT = TRUE;
     40  }
     41
    3642  if ((N = get_argument (argc, argv, "-create-id"))) {
    3743    options.mode = ADDSTAR_MODE_CREATE_ID;
    … …  
    350356  // XXX for the moment, make this selection manual.  it needs to be automatic
    351357  // based on the state of the SkyTable
     358  HOST_ID = 0;
     359  HOSTDIR = NULL;
    352360  PARALLEL = FALSE;
    353361  if ((N = get_argument (argc, argv, "-parallel"))) {
    … …  
    466474}
    467475
     476int strextend (char *input, char *format,...) {
     477
     478  char tmpextra[1024], tmpline[1024];
     479  va_list argp;
     480
     481  va_start (argp, format);
     482  vsnprintf (tmpextra, 1024, format, argp);
     483  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
     484  strcpy (input, tmpline);
     485
     486  return TRUE;
     487}
     488
    468489/** addstar modes:
    469490 

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/args_parallel_client.c

    r33963 r37089  
    8787  }
    8888
     89  OLD_RESORT = FALSE;
     90  if ((N = get_argument (argc, argv, "-old-resort"))) {
     91    remove_argument (N, &argc, argv);
     92    OLD_RESORT = TRUE;
     93  }
     94
    8995  /* extra error messages */
    9096  VERBOSE = FALSE;
    … …  
    118124  exit (2);
    119125}
     126
     127int strextend (char *input, char *format,...) {
     128
     129  char tmpextra[1024], tmpline[1024];
     130  va_list argp;
     131
     132  va_start (argp, format);
     133  vsnprintf (tmpextra, 1024, format, argp);
     134  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
     135  strcpy (input, tmpline);
     136
     137  return TRUE;
     138}
     139

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/resort_catalog.c

    r37036 r37089  
    77void resort_catalog_old (Catalog *catalog) {
    88
    9   off_t *next_meas;
    10   off_t Naves, Nmeas;
     9  off_t *next_meas, *next_lens;
     10  off_t Naves, Nmeas, Nlens;
    1111
    1212  if (catalog[0].sorted == TRUE) return;
    … …  
    1616  /* internal counters */
    1717  Nmeas = catalog[0].Nmeasure;
     18  Nlens = catalog[0].Nlensing;
    1819  Naves = catalog[0].Naverage;
    1920 
    2021  /* set up pointers for linked list of measure, missing */
    2122  next_meas = build_measure_links (catalog[0].average, Naves, catalog[0].measure, Nmeas);
     23  next_lens = build_lensing_links (catalog[0].average, Naves, catalog[0].lensing, Nlens);
    2224
    2325  catalog[0].sorted = TRUE;
    2426  catalog[0].measure = sort_measure (catalog[0].average, Naves, catalog[0].measure, Nmeas, next_meas);
     27  catalog[0].lensing = sort_lensing (catalog[0].average, Naves, catalog[0].lensing, Nlens, next_lens);
    2528
    2629  MARKTIME ("  match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeas, (long long) Naves);

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/resort_threaded.c

    r37036 r37089  
    1313  int iThread;
    1414  int state;
    15   int nosort;
     15  int forcesort;
    1616  char *filename;
    1717  SkyRegion *region;
    … …  
    3737    }
    3838
     39    // chose the catalog file (local or remote?)
     40    char hostfile[1024];
     41    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, threadData->region->name);
     42    catalog.filename  = HOST_ID ? hostfile : threadData->filename;
     43
    3944    // set the parameters which guide catalog open/load/create
    40     catalog.filename  = threadData->filename;
    4145    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    4246    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
    … …  
    6468
    6569    // this is an overloaded value to mean 'force sort'
    66     if (threadData->nosort == 3) catalog.sorted = FALSE;
     70    if (threadData->forcesort) catalog.sorted = FALSE;
    6771
    6872    if (OLD_RESORT) {
    … …  
    8892}
    8993
    90 int resort_threaded (AddstarClientOptions *options, SkyTable *sky) {
     94int resort_threaded (SkyList *skylist, int ForceSort) {
    9195
    9296  int j, launched, done, Nwait, Nrun;
    … …  
    97101  ThreadData *threadData;
    98102  pthread_t *threads;
    99 
    100   double dtime;
    101   struct timeval start, stop;
    102   gettimeofday (&start, NULL);
    103 
    104   SkyList *skylist = SkyListByPatch (sky, -1, &UserPatch);
    105  
    106   // in these cases, limit the sky catalogs to an existing subset
    107   if (options->existing_regions) {
    108     SkyList *tmp;
    109     tmp = SkyListExistingSubset (skylist, CATDIR);
    110     SkyListFree (skylist);
    111     skylist = tmp;
    112   }
    113   if (VERBOSE) fprintf (stderr, "writing to "OFF_T_FMT" regions\n",  skylist[0].Nregions);
    114 
    115   if (options->only_images) {
    116     fprintf (stderr, "-image (only images) makes no sense with -resort\n");
    117     exit (2);
    118   }
    119   if (options->only_match) {
    120     fprintf (stderr, "-only-match makes no sense with -resort\n");
    121     exit (2);
    122   }
    123   if (options->nosort == 1) {
    124     fprintf (stderr, "-nosort makes no sense with -resort\n");
    125     exit (2);
    126   }
    127   if (options->update) {
    128     fprintf (stderr, "-update makes no sense with -resort\n");
    129     exit (2);
    130   }
    131   if (options->calibrate) {
    132     fprintf (stderr, "-cal (calibrate) makes no sense with -resort\n");
    133     exit (2);
    134   }
    135103
    136104  ALLOCATE(threads, pthread_t, NTHREADS);
    … …  
    148116  Naverage = Nmeasure = 0;
    149117  for (i = 0; i < skylist[0].Nregions; i++) {
     118
     119    // does this host ID match the desired location for the table?
     120    if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
    150121
    151122    launched = FALSE;
    … …  
    166137        if (threadData[j].state == TS_FAIL) goto failure;
    167138        if (threadData[j].state != TS_WAIT) continue;
    168         threadData[j].nosort = options->nosort;
     139        threadData[j].forcesort = ForceSort;
    169140        threadData[j].filename = skylist[0].filename[i];
    170141        threadData[j].region = skylist[0].regions[i];
    … …  
    203174  }
    204175
    205   gettimeofday (&stop, NULL);
    206   dtime = DTIME (stop, start);
    207   fprintf (stderr, "SUCCESS: elapsed time %9.4f sec for, "OFF_T_FMT" average, "OFF_T_FMT" measure\n", dtime, Naverage, Nmeasure);
    208 
    209176  // for (i = 0; i < NTHREADS; i++) {
    210177  //   pthread_cancel(threads[i]);

  • branches/eam_branches/ipp-20140717/Ohana/src/addstar/src/resort_unthreaded.c

    r35142 r37089  
    11# include "addstar.h"
    22
    3 int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath) {
     3// pass in options and skylist?
     4int resort_unthreaded (SkyList *skylist, int ForceSort) {
    45
    5   double dtime;
    6   struct timeval start, stop;
     6  off_t i;
     7  off_t Naverage, Nmeasure;
     8  Catalog catalog;
    79
    8   gettimeofday (&start, NULL);
     10  /* match stars to existing catalog data (or otherwise manipulate catalog data) */
     11  Naverage = Nmeasure = 0;
     12  for (i = 0; i < skylist[0].Nregions; i++) {
    913
    10   SkyList *skylist = SkyListByPatch (sky, -1, &UserPatch);
    11  
    12   // in these cases, limit the sky catalogs to an existing subset
    13   if (options->existing_regions) {
    14     SkyList *tmp;
    15     tmp = SkyListExistingSubset (skylist, CATDIR);
    16     SkyListFree (skylist);
    17     skylist = tmp;
     14    // does this host ID match the desired location for the table?
     15    if (!HostTableTestHost(skylist[0].regions[i], HOST_ID)) continue;
     16
     17    // chose the catalog file (local or remote?)
     18    char hostfile[1024];
     19    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
     20    catalog.filename  = HOST_ID ? hostfile : skylist[0].filename[i];
     21
     22    // set the parameters which guide catalog open/load/create
     23    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
     24    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
     25    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_LENSING;
     26    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
     27
     28    // an error exit status here is a significant error (disk I/O or file access)
     29    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
     30      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename);
     31      exit (2);
     32    }
     33
     34    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
     35    if (catalog.Naves_disk == 0) {
     36      dvo_catalog_unlock (&catalog);
     37      dvo_catalog_free (&catalog);
     38      continue;
     39    }
     40
     41    // this is an overloaded value to mean 'force sort'
     42    if (ForceSort) catalog.sorted = FALSE;
     43
     44    if (OLD_RESORT) {
     45      resort_catalog_old (&catalog);
     46    } else {
     47      resort_catalog (&catalog);
     48    }
     49
     50    // report total updated values
     51    Naverage += catalog.Naverage;
     52    Nmeasure += catalog.Nmeasure;
     53
     54    // write out catalog, if appropriate
     55    SetProtect (TRUE);
     56    dvo_catalog_save (&catalog, VERBOSE);
     57    SetProtect (FALSE);
     58    dvo_catalog_unlock (&catalog);
     59    dvo_catalog_free (&catalog);
    1860  }
    19   if (VERBOSE) fprintf (stderr, "writing to "OFF_T_FMT" regions\n",  skylist[0].Nregions);
    20 
    21   if (options->only_images) {
    22     fprintf (stderr, "-image (only images) makes no sense with -resort\n");
    23     exit (2);
    24   }
    25   if (options->only_match) {
    26     fprintf (stderr, "-only-match makes no sense with -resort\n");
    27     exit (2);
    28   }
    29   if (options->nosort == 1) {
    30     fprintf (stderr, "-nosort makes no sense with -resort\n");
    31     exit (2);
    32   }
    33   if (options->update) {
    34     fprintf (stderr, "-update makes no sense with -resort\n");
    35     exit (2);
    36   }
    37   if (options->calibrate) {
    38     fprintf (stderr, "-cal (calibrate) makes no sense with -resort\n");
    39     exit (2);
    40   }
    41 
    42   resort_unthreaded_catalogs (options, skylist, hostID, hostpath);
    43 
    44   gettimeofday (&stop, NULL);
    45   dtime = DTIME (stop, start);
    46   fprintf (stderr, "SUCCESS: elapsed time %9.4f sec\n", dtime);
    47 
    4861  return TRUE;
    4962}
    5063
     64
Note: See TracChangeset for help on using the changeset viewer.