IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38687


Ignore:
Timestamp:
Aug 28, 2015, 4:03:30 PM (11 years ago)
Author:
eugene
Message:

allow AstromOffsetMapLoad to read in small segments at a time (to save memory)

Location:
branches/eam_branches/ipp-20150625/Ohana/src/libdvo
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20150625/Ohana/src/libdvo/Makefile

    r38472 r38687  
    66HOME    =       $(ROOT)/src/libdvo
    77AUTO    =       $(ROOT)/src/libautocode
     8BIN     =       $(HOME)/bin
    89LIB     =       $(HOME)/lib
    910SRC     =       $(HOME)/src
    … …  
    2223FULL_LDFLAGS  = $(BASE_LDFLAGS) -lFITS -lohana
    2324
    24 default: install
     25TEST_CFLAGS   = $(BASE_CFLAGS)
     26TEST_CPPFLAGS = $(BASE_CPPFLAGS)
     27TEST_LDFLAGS  = $(BASE_LDFLAGS) -ldvo -lFITS -lohana -ltap_ohana
     28
     29TESTPROG = tap_AstromOffsetMapIO
     30$(TESTPROG) : % : $(TESTBIN)/%
     31test: $(TESTPROG)
     32        for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS); done
     33test.verbose: $(TESTPROG)
     34        for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 |& $(TESTHARNESS) -v; done
     35
    2536install: $(DESTLIB)/libdvo.a $(DESTLIB)/libdvo.$(DLLTYPE)
    2637libdvo: $(LIB)/libdvo.$(ARCH).a $(LIB)/libdvo.$(ARCH).$(DLLTYPE)
    … …  
    183194        cd $(AUTO) && $(MAKE) clean
    184195
    185 TESTPROG =
    186 $(TESTPROG) : % : $(TESTBIN)/% $(OBJS)
    187 test: $(TESTPROG)
    188         for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 | $(TESTHARNESS); done
    189 test.verbose: $(TESTPROG)
    190         for i in $(TESTPROG); do $(TESTBIN)/$$i 2>&1 |& $(TESTHARNESS) -v; done
    191 
    192 .PRECIOUS: $(ASRC)/%.c
     196# .PRECIOUS: $(ASRC)/%.c

  • branches/eam_branches/ipp-20150625/Ohana/src/libdvo/include/libdvo_astro.h

    r38462 r38687  
    160160
    161161/* in AstromOffsetMapIO.c */
    162 AstromOffsetTable *AstromOffsetMapLoad (char *filename, int VERBOSE);
     162AstromOffsetTable *AstromOffsetMapLoad (char *filename, int Nrows, int VERBOSE);
    163163int AstromOffsetMapSave (AstromOffsetTable *table, char *filename);
    164164AstromOffsetTable *AstromOffsetMapToTable(AstromOffsetMap_Disk_6x6 *map_disk, off_t Nmap);
    165165AstromOffsetMap_Disk_6x6 *AstromOffsetTableToMap(AstromOffsetTable *table, off_t *Nmap);
     166int AstromOffsetTableSetIDs (AstromOffsetTable *table);
     167AstromOffsetTable *AstromOffsetMapAppendToTable(AstromOffsetTable *table, AstromOffsetMap_Disk_6x6 *map_disk, off_t Nmap);
    166168
    167169/* in AstromOffsetMapOps.c */

  • branches/eam_branches/ipp-20150625/Ohana/src/libdvo/src/AstromOffsetMapIO.c

    r38441 r38687  
    11# include "dvo.h"
    22
    3 AstromOffsetTable *AstromOffsetMapLoad (char *filename, int VERBOSE) {
     3AstromOffsetTable *AstromOffsetMapLoad (char *filename, int Nrows, int VERBOSE) {
    44
    55  off_t Nmap;
    … …  
    77  Header header;
    88  Matrix matrix;
     9
    910  Header theader;
    1011  FTable ftable;
     12  ftable.header = &theader;
    1113
    1214  FILE *f = fopen (filename, "r");
    … …  
    1719
    1820  /* load in table data */
    19   ftable.header = &theader;
    2021  if (!gfits_fread_header (f, &header)) {
    2122    if (VERBOSE) fprintf (stderr, "can't read Astrom Offset Map header\n");
    … …  
    3031  }
    3132
    32   // for now, we only have one flavor (6x6)
    33 
    34   if (!gfits_fread_ftable (f, &ftable, "ASTROM_OFFSET_MAP_DISK_6x6")) {
    35     if (VERBOSE) fprintf (stderr, "can't read Astrom Offset Map table\n");
    36     gfits_free_header (&header);
    37     gfits_free_matrix (&matrix);
    38     fclose (f);
    39     return (NULL);
    40   }
    41   AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL);
    42   if (!map_disk) {
    43     fprintf (stderr, "ERROR: failed to read Astrom Offset Map\n");
    44     exit (2);
    45   }
    46 
    47   // AstromOffsetMap_Disk_6x6 *map_disk is an external format stored as an array of maps.
    48   // Convert the disk array of maps to then internal format in a rich structure:
    49   AstromOffsetTable *table = AstromOffsetMapToTable (map_disk, Nmap);
     33  // the output table is not what is stored in the FITS file.  read and convert
     34  // from the disk file format
     35  AstromOffsetTable *table = NULL;
     36
     37  // for now, we only have one flavor (6x6) of disk file format
     38
     39  // loop over the ftable reading blocks of 100k rows at a time
     40  if (Nrows) {
     41    if (!gfits_find_Xheader (f, ftable.header, "ASTROM_OFFSET_MAP_DISK_6x6")) myAbort ("problem 1");
     42    int NrowsTotal = ftable.header[0].Naxis[1];
     43    int Nblocks = (NrowsTotal % Nrows) ? (int) (NrowsTotal / Nrows) + 1 : (NrowsTotal / Nrows);
     44
     45    int i;
     46    for (i = 0; i < Nblocks; i++) {
     47      if (!gfits_fread_ftable_range (f, FALSE, &ftable, i*Nrows, Nrows)) myAbort ("problem 2");
     48   
     49      AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL);
     50      if (!map_disk) myAbort ("ERROR: failed to read Astrom Offset Map\n");
     51
     52      table = AstromOffsetMapAppendToTable (table, map_disk, Nmap);
     53      gfits_free_table  (&ftable);
     54    }
     55    AstromOffsetTableSetIDs (table);
     56    gfits_free_header (ftable.header);
     57  } else {
     58
     59    // convert the blocks into the equivalent table entries and append
     60    if (!gfits_fread_ftable (f, &ftable, "ASTROM_OFFSET_MAP_DISK_6x6")) {
     61      if (VERBOSE) fprintf (stderr, "can't read Astrom Offset Map table\n");
     62      gfits_free_header (&header);
     63      gfits_free_matrix (&matrix);
     64      fclose (f);
     65      return (NULL);
     66    }
     67    AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL);
     68    if (!map_disk) {
     69      fprintf (stderr, "ERROR: failed to read Astrom Offset Map\n");
     70      exit (2);
     71    }
     72
     73    // AstromOffsetMap_Disk_6x6 *map_disk is an external format stored as an array of maps.
     74    // Convert the disk array of maps to then internal format in a rich structure:
     75    table = AstromOffsetMapToTable (map_disk, Nmap);
     76    gfits_free_table  (&ftable);
     77  }
    5078
    5179  gfits_free_header (&theader);
    5280  gfits_free_matrix (&matrix);
    53   gfits_free_table  (&ftable);
    5481
    5582  return (table);
    … …  
    139166   
    140167    // STORE THESE VALUES?
    141     table->map[i][0].dY           = map_disk[i].dX;
    142     table->map[i][0].dX           = map_disk[i].dY;
     168    table->map[i][0].dX           = map_disk[i].dX;
     169    table->map[i][0].dY           = map_disk[i].dY;
    143170
    144171    // since this was on disk, we obviously keep it
    … …  
    182209    map_disk[Ndisk].dY      = table->map[i][0].dY;
    183210
    184    
    185 
    186211    for (j = 0; j < map_disk[Ndisk].Nx; j++) {
    187212      for (k = 0; k < map_disk[Ndisk].Ny; k++) {
    … …  
    206231}
    207232
     233AstromOffsetTable *AstromOffsetMapAppendToTable(AstromOffsetTable *table, AstromOffsetMap_Disk_6x6 *map_disk, off_t Nmap) {
     234
     235  int i, j, k, Nstart;
     236
     237  if (table == NULL) {
     238    ALLOCATE (table, AstromOffsetTable, 1);
     239    Nstart = 0;
     240    table->Nmap = Nmap;
     241    ALLOCATE (table->map, AstromOffsetMap *, Nmap);
     242  } else {
     243    Nstart = table->Nmap;
     244    table->Nmap += Nmap;
     245    REALLOCATE (table->map, AstromOffsetMap *, table->Nmap);
     246  }
     247
     248  // append map values (this allocates just the area needed for each image, not the
     249  // full 6x6, saving some memory while doing the analysis)
     250  for (i = 0; i < Nmap; i++) {
     251    int N = i + Nstart;
     252    ALLOCATE (table->map[N], AstromOffsetMap, 1);
     253
     254    table->map[N][0].Nx           = map_disk[i].Nx;
     255    table->map[N][0].Ny           = map_disk[i].Ny;
     256    table->map[N][0].tableID      = map_disk[i].tableID;
     257    table->map[N][0].imageID      = map_disk[i].imageID;
     258   
     259    // STORE THESE VALUES?
     260    table->map[N][0].dX           = map_disk[i].dX;
     261    table->map[N][0].dY           = map_disk[i].dY;
     262
     263    // since this was on disk, we obviously keep it
     264    table->map[N][0].keep         = TRUE;
     265
     266    ALLOCATE (table->map[N][0].dXv, float *, map_disk[i].Nx);
     267    ALLOCATE (table->map[N][0].dYv, float *, map_disk[i].Nx);
     268
     269    for (j = 0; j < map_disk[i].Nx; j++) {
     270      ALLOCATE (table->map[N][0].dXv[j], float, map_disk[i].Ny);
     271      ALLOCATE (table->map[N][0].dYv[j], float, map_disk[i].Ny);
     272
     273      for (k = 0; k < map_disk[i].Ny; k++) {
     274        table->map[N][0].dXv[j][k] = map_disk[i].dXv[j][k];
     275        table->map[N][0].dYv[j][k] = map_disk[i].dYv[j][k];
     276      }
     277    }
     278  }
     279  return table;
     280}
     281
     282int AstromOffsetTableSetIDs (AstromOffsetTable *table) {
     283
     284  int i;
     285
     286  // find the max value of imageID
     287  int MaxTableID = 0;
     288  int MaxImageID = 0;
     289  for (i = 0; i < table->Nmap; i++) {
     290    MaxTableID = MAX(table->map[i][0].tableID, MaxTableID);
     291    MaxImageID = MAX(table->map[i][0].imageID, MaxImageID);
     292  }
     293  table->MaxTableID = MaxTableID;
     294  table->MaxImageID = MaxImageID;
     295
     296  // generate the index and init values to -1
     297  ALLOCATE (table->imageIDtoTableSeq, int, MaxImageID + 1);
     298  for (i = 0; i <= MaxImageID; i++) {
     299    table->imageIDtoTableSeq[i] = -1;
     300  }
     301
     302  // assign the ID values
     303  for (i = 0; i < table->Nmap; i++) {
     304    int ImageID = table->map[i][0].imageID;
     305    myAssert (table->imageIDtoTableSeq[ImageID] == -1, "oops, duplicate image IDs");
     306    table->imageIDtoTableSeq[ImageID] = i;
     307  }
     308  return TRUE;
     309}

  • branches/eam_branches/ipp-20150625/Ohana/src/libdvo/src/LoadImages.c

    r38553 r38687  
    8585
    8686  if (table) AstromOffsetTableFree(table);
    87   table = AstromOffsetMapLoad (mapfile, FALSE);
     87  table = AstromOffsetMapLoad (mapfile, 100000, FALSE);
    8888
    8989  // assign images.coords.offsetMap -> table->map[i]
Note: See TracChangeset for help on using the changeset viewer.