IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Feb 26, 2016, 8:31:22 PM (10 years ago)
Author:
eugene
Message:

working on very pedantic gcc to track down memory corruption issues

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20160226/src/libfits/table/F_compress_T.c

    r39358 r39409  
    2323# define ESCAPE { fprintf (stderr, "error in %s @ line %d\n", __func__, __LINE__); goto escape; }
    2424
    25 int gfits_collect_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows);
    26 int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows);
     25int gfits_collect_table_data (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows);
     26int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows);
    2727
    2828static float timeSum1 = 0.0;
    … …  
    5151}
    5252
    53 int gfits_compress_table (FTable *srctable, FTable *tgttable, int ztilelen, char *zcmptype) {
    54 
    55   int i, j;
     53int gfits_compress_table (FTable *srctable, FTable *tgttable, unsigned long int ztilelen, char *zcmptype) {
    5654
    5755  char keyword[81];
    … …  
    6967  gettimeofday (&startTimer, (void *) NULL);
    7068
    71   int Ntile, ztilelast;
     69  unsigned long int Ntile, ztilelast;
    7270  if (!ztilelen) ztilelen = srcheader->Naxis[1];
    7371
    … …  
    104102
    105103  ALLOCATE_ZERO (fields, TableField, Nfields);
    106   for (i = 0; i < Nfields; i++) {
    107     snprintf (keyword, 80, "TTYPE%d", i+1);
    108     if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].ttype)) ESCAPE;
    109     if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].ttype_cmt)) ESCAPE;
     104
     105  int field;
     106  for (field = 0; field < Nfields; field++) {
     107    snprintf (keyword, 80, "TTYPE%d", field+1);
     108    if (!gfits_scan (srcheader, keyword, "%s", 1, fields[field].ttype)) ESCAPE;
     109    if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[field].ttype_cmt)) ESCAPE;
    110110
    111111    // TUNIT is not mandatory
    112     snprintf (keyword, 80, "TUNIT%d", i+1);
    113     if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tunit)) {
    114       fields[i].tunit[0] = 0;
     112    snprintf (keyword, 80, "TUNIT%d", field+1);
     113    if (!gfits_scan (srcheader, keyword, "%s", 1, fields[field].tunit)) {
     114      fields[field].tunit[0] = 0;
    115115    } else {
    116       if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tunit_cmt)) ESCAPE;
     116      if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[field].tunit_cmt)) ESCAPE;
    117117    }
    118118   
    119     snprintf (keyword, 80, "TFORM%d", i+1);
    120     if (!gfits_scan (srcheader, keyword, "%s", 1, fields[i].tformat)) ESCAPE;
    121     if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[i].tformat_cmt)) ESCAPE;
     119    snprintf (keyword, 80, "TFORM%d", field+1);
     120    if (!gfits_scan (srcheader, keyword, "%s", 1, fields[field].tformat)) ESCAPE;
     121    if (!gfits_scan_alt (srcheader, keyword, "%C", 1, fields[field].tformat_cmt)) ESCAPE;
    122122
    123123    // for now we set all fields to the requested type.  since I do not yet know the column data types I cannot yet assign automatic cmptypes
    124     strcpy (fields[i].zctype, zcmptype);
     124    strcpy (fields[field].zctype, zcmptype);
    125125
    126126    // by using "P" format, we are limited to 32bit pointers (2GB heap)
    127     if (!gfits_define_bintable_column (tgtheader, "1QB(0)", fields[i].ttype, fields[i].ttype_cmt, fields[i].tunit, 1.0, 0.0)) ESCAPE;
     127    if (!gfits_define_bintable_column (tgtheader, "1QB(0)", fields[field].ttype, fields[field].ttype_cmt, fields[field].tunit, 1.0, 0.0)) ESCAPE;
    128128  }   
    129129
    … …  
    137137  char *tmpbuffer = NULL;
    138138  ALLOCATE_ZERO (tmpbuffer, char, 16*Ntile); // need space for Ntile entries, each of width 16 bytes (2 long)
    139   for (i = 0; i < Nfields; i++) {
    140     if (!gfits_set_bintable_column (tgtheader, tgttable, fields[i].ttype, tmpbuffer, Ntile)) ESCAPE;
     139  for (field = 0; field < Nfields; field++) {
     140    if (!gfits_set_bintable_column (tgtheader, tgttable, fields[field].ttype, tmpbuffer, Ntile)) ESCAPE;
    141141  }
    142142  free (tmpbuffer);
    … …  
    146146
    147147  // define compression-specific keywords, update header as needed.
    148   if (!gfits_modify (tgtheader, "ZTILELEN", "%d", 1, ztilelen)) ESCAPE;
     148  if (!gfits_modify (tgtheader, "ZTILELEN", "%lud", 1, ztilelen)) ESCAPE;
    149149
    150150  if (!gfits_modify (tgtheader, "ZNAXIS1", OFF_T_FMT, 1, srcheader->Naxis[0])) ESCAPE;
    … …  
    163163  int offset = 0; // bytes from first column of first field to the current field (accumulate to set)
    164164
    165   for (i = 0; i < Nfields; i++) {
    166     snprintf (keyword, 81, "ZFORM%d", i+1);
    167     if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[i].tformat)) ESCAPE;
    168     if (!gfits_varlength_column_define (tgttable, &fields[i].zdef, i+1)) ESCAPE;
    169     if (!gfits_bintable_format (fields[i].tformat, fields[i].datatype, &fields[i].Nvalues, &fields[i].pixsize)) ESCAPE; //
    170     fields[i].rowsize = fields[i].Nvalues*fields[i].pixsize;
    171     max_width = MAX(max_width, fields[i].rowsize);
    172 
    173     if (!strcasecmp (fields[i].zctype, "AUTO")) {
    174       if (!strcmp (fields[i].datatype, "short") ||
    175           !strcmp (fields[i].datatype, "float") ||
    176           !strcmp (fields[i].datatype, "double")||
    177           !strcmp (fields[i].datatype, "int64_t")) {
    178         strcpy (fields[i].zctype, "GZIP_2");
     165  for (field = 0; field < Nfields; field++) {
     166    snprintf (keyword, 81, "ZFORM%d", field+1);
     167    if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[field].tformat)) ESCAPE;
     168    if (!gfits_varlength_column_define (tgttable, &fields[field].zdef, field+1)) ESCAPE;
     169    if (!gfits_bintable_format (fields[field].tformat, fields[field].datatype, &fields[field].Nvalues, &fields[field].pixsize)) ESCAPE; //
     170    fields[field].rowsize = fields[field].Nvalues*fields[field].pixsize;
     171    max_width = MAX(max_width, fields[field].rowsize);
     172
     173    if (!strcasecmp (fields[field].zctype, "AUTO")) {
     174      if (!strcmp (fields[field].datatype, "short") ||
     175          !strcmp (fields[field].datatype, "float") ||
     176          !strcmp (fields[field].datatype, "double")||
     177          !strcmp (fields[field].datatype, "int64_t")) {
     178        strcpy (fields[field].zctype, "GZIP_2");
    179179        goto got_cmptype;
    180180      }
    181       if (!strcmp (fields[i].datatype, "int")) {
    182         strcpy (fields[i].zctype, "RICE_1");
     181      if (!strcmp (fields[field].datatype, "int")) {
     182        strcpy (fields[field].zctype, "RICE_1");
    183183        goto got_cmptype;
    184184      }
    185       if (!strcmp (fields[i].datatype, "byte") ||
    186           !strcmp (fields[i].datatype, "char")) {
    187         strcpy (fields[i].zctype, "GZIP_1");
     185      if (!strcmp (fields[field].datatype, "byte") ||
     186          !strcmp (fields[field].datatype, "char")) {
     187        strcpy (fields[field].zctype, "GZIP_1");
    188188        goto got_cmptype;
    189189      }
    … …  
    193193
    194194    // OVERRIDE: RICE can only be used on integer fields
    195     if (!strcasecmp (fields[i].zctype, "RICE_1") || !strcasecmp (fields[i].zctype, "RICE_ONE")) {
    196       if (!strcmp (fields[i].datatype, "float") || !strcmp (fields[i].datatype, "double") || !strcmp (fields[i].datatype, "int64_t")) {
    197         strcpy (fields[i].zctype, "GZIP_2");
     195    if (!strcasecmp (fields[field].zctype, "RICE_1") || !strcasecmp (fields[field].zctype, "RICE_ONE")) {
     196      if (!strcmp (fields[field].datatype, "float") || !strcmp (fields[field].datatype, "double") || !strcmp (fields[field].datatype, "int64_t")) {
     197        strcpy (fields[field].zctype, "GZIP_2");
    198198      }
    199199    }
    200200
    201201    // OVERRIDE: GZIP_2 invalid for B (use GZIP_1)
    202     if (!strcasecmp (fields[i].zctype, "GZIP_2") && !strcmp (fields[i].datatype, "byte")) {
    203       strcpy (fields[i].zctype, "GZIP_1");
     202    if (!strcasecmp (fields[field].zctype, "GZIP_2") && !strcmp (fields[field].datatype, "byte")) {
     203      strcpy (fields[field].zctype, "GZIP_1");
    204204    }
    205205
    206206    // compression type per column (XXX for now we are just using zcmptype, as set above)
    207     snprintf (keyword, 81, "ZCTYP%d", i+1);
    208     if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[i].zctype)) ESCAPE;
    209 
    210     fields[i].offset = offset;
    211     offset += fields[i].rowsize;
     207    snprintf (keyword, 81, "ZCTYP%d", field+1);
     208    if (!gfits_modify (tgtheader, keyword, "%s", 1, fields[field].zctype)) ESCAPE;
     209
     210    fields[field].offset = offset;
     211    offset += fields[field].rowsize;
    212212  }
    213213
    214214  // allocate the intermediate storage buffers
    215   int Nzdata_alloc = 2*max_width*ztilelen + 100;
     215  unsigned long int Nzdata_alloc = 2*max_width*ztilelen + 100;
    216216  ALLOCATE (raw,   char, max_width*ztilelen);
    217217  ALLOCATE (zdata, char, Nzdata_alloc);
    … …  
    229229  // compress the data : copy into a tile, compress the tile, then add to the output table
    230230  // each tile -> 1 row of the output table
    231   for (i = 0; i < Ntile; i++) {
    232 
    233     for (j = 0; j < Nfields; j++) {
    234 
    235       gettimeofday (&startTimer, (void *) NULL);
    236 
    237       int Nrows = (i == Ntile - 1) ? ztilelast : ztilelen;
    238       int row_start = i*ztilelen;
     231  unsigned long int tile;
     232  for (tile = 0; tile < Ntile; tile++) {
     233
     234    for (field = 0; field < Nfields; field++) {
     235
     236      gettimeofday (&startTimer, (void *) NULL);
     237
     238      unsigned long int Nrows = (tile == Ntile - 1) ? ztilelast : ztilelen;
     239      unsigned long int row_start = tile*ztilelen;
    239240      // ^- first row for this tile & field
    240241      // NOTE: assumes each tile has the same length, ex the last (true for now)
    241242
    242243      // copy the raw pixels from their native matrix locations to the temporary output buffer
    243       if (!strcasecmp(fields[j].zctype, "GZIP_2") || !strcasecmp(fields[j].zctype, "NONE_2")) {
    244         if (!gfits_collect_table_gzp2 (srctable, &fields[j], raw, row_start, Nrows)) ESCAPE;
     244      if (!strcasecmp(fields[field].zctype, "GZIP_2") || !strcasecmp(fields[field].zctype, "NONE_2")) {
     245        if (!gfits_collect_table_gzp2 (srctable, &fields[field], raw, row_start, Nrows)) ESCAPE;
    245246      } else {
    246         if (!gfits_collect_table_data (srctable, &fields[j], raw, row_start, Nrows)) ESCAPE;
     247        if (!gfits_collect_table_data (srctable, &fields[field], raw, row_start, Nrows)) ESCAPE;
    247248      }
    248249     
    … …  
    256257 
    257258      if (VERBOSE_DUMP) {
    258         int k;
     259        unsigned long int k;
    259260        fprintf (stderr, "c1: ");
    260         for (k = 0; k < Nrows*fields[j].Nvalues*fields[j].pixsize; k++) {
     261        for (k = 0; k < Nrows*fields[field].Nvalues*fields[field].pixsize; k++) {
    261262          fprintf (stderr, "%02hhx", raw[k]);
    262263          if (k % 2) fprintf (stderr, " ");
    … …  
    269270      // optname, optvalue = NULL, Noptions = 0
    270271     
    271       int Nraw = Nrows*fields[j].Nvalues; // number of pixels
    272       if (!strcasecmp(fields[j].zctype, "GZIP_1")) {
    273         if (!gfits_byteswap_zdata (raw, Nraw * fields[j].pixsize, fields[j].pixsize)) ESCAPE;
     272      unsigned long int Nraw = Nrows*fields[field].Nvalues; // number of pixels
     273      if (!strcasecmp(fields[field].zctype, "GZIP_1")) {
     274        if (!gfits_byteswap_zdata (raw, Nraw * fields[field].pixsize, fields[field].pixsize)) ESCAPE;
    274275      }
    275276
    … …  
    281282 
    282283      if (VERBOSE_DUMP) {
    283         int k;
     284        unsigned long int k;
    284285        fprintf (stderr, "c2: ");
    285         for (k = 0; k < Nrows*fields[j].Nvalues*fields[j].pixsize; k++) {
     286        for (k = 0; k < Nrows*fields[field].Nvalues*fields[field].pixsize; k++) {
    286287          fprintf (stderr, "%02hhx", raw[k]);
    287288          if (k % 2) fprintf (stderr, " ");
    … …  
    291292      }
    292293
    293       int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression
    294       if (!gfits_compress_data (zdata, &Nzdata, fields[j].zctype, NULL, NULL, 0, raw, Nraw, fields[j].pixsize, 0, 0)) ESCAPE;
     294      unsigned long int Nzdata = Nzdata_alloc; // available space, replaced with actual output size on compression
     295      if (!gfits_compress_data (zdata, &Nzdata, fields[field].zctype, NULL, NULL, 0, raw, Nraw, fields[field].pixsize, 0, 0)) ESCAPE;
    295296     
    296297      // XXX TIMER 2c
    … …  
    301302 
    302303      if (VERBOSE_DUMP) {
    303         int k;
     304        unsigned long int k;
    304305        fprintf (stderr, "c3: ");
    305306        for (k = 0; k < Nzdata; k++) {
    … …  
    311312      }
    312313
    313       if (strcasecmp(fields[j].zctype, "NONE_2") && // NONE and NONE_1 not swapped?
    314           strcasecmp(fields[j].zctype, "GZIP_1") &&
    315           strcasecmp(fields[j].zctype, "GZIP_2") &&
    316           strcasecmp(fields[j].zctype, "RICE_1") &&
    317           strcasecmp(fields[j].zctype, "RICE_ONE")) {
    318         if (!gfits_byteswap_zdata (zdata, Nzdata, fields[j].pixsize)) ESCAPE;
     314      if (strcasecmp(fields[field].zctype, "NONE_2") && // NONE and NONE_1 not swapped?
     315          strcasecmp(fields[field].zctype, "GZIP_1") &&
     316          strcasecmp(fields[field].zctype, "GZIP_2") &&
     317          strcasecmp(fields[field].zctype, "RICE_1") &&
     318          strcasecmp(fields[field].zctype, "RICE_ONE")) {
     319        if (!gfits_byteswap_zdata (zdata, Nzdata, fields[field].pixsize)) ESCAPE;
    319320      }
    320321
    … …  
    326327 
    327328      if (VERBOSE_DUMP) {
    328         int k;
     329        unsigned long int k;
    329330        fprintf (stderr, "c4: ");
    330331        for (k = 0; k < Nzdata; k++) {
    … …  
    336337      }
    337338
    338       if (!gfits_varlength_column_add_data (tgttable, zdata, Nzdata, i, &fields[j].zdef)) ESCAPE;
     339      if (!gfits_varlength_column_add_data (tgttable, zdata, Nzdata, field, &fields[field].zdef)) ESCAPE;
    339340      // XXX TIMER 2e
    340341      gettimeofday (&stopTimer, (void *) NULL);
    … …  
    347348  if (OHANA_MEMCHECK) ohana_memcheck (TRUE);
    348349
    349   for (i = 0; i < Nfields; i++) {
    350     if (!gfits_varlength_column_finish (tgttable, &fields[i].zdef)) ESCAPE;
     350  for (field = 0; field < Nfields; field++) {
     351    if (!gfits_varlength_column_finish (tgttable, &fields[field].zdef)) ESCAPE;
    351352  }
    352353
    … …  
    375376// raw_pixsize is the bytes / pixel for the input matrix
    376377// place the raw image bytes for the current tile into the tile buffer
    377 int gfits_collect_table_data (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
    378 
    379   off_t i;
     378int gfits_collect_table_data (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows) {
     379
     380  unsigned long int i;
    380381
    381382  // we are copying NN rows into the column which starts at XX and has MM bytes per row
    … …  
    399400}
    400401
    401 int gfits_collect_table_data_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
    402 
    403   off_t i;
     402int gfits_collect_table_data_alt (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows) {
     403
     404  unsigned long int i;
    404405
    405406  // we are copying NN rows of the column which starts at XX and has MM bytes per row
    … …  
    420421}
    421422
    422 int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
    423 
    424   off_t i, j, k;
     423int gfits_collect_table_gzp2 (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows) {
     424
     425  off_t j, k;
    425426
    426427  // we are copying NN rows into the column which starts at XX and has MM bytes per row
    … …  
    439440    char *tblptr_start = &table->buffer[Nx*row_start + offset + k];
    440441# endif
     442    unsigned long int i;
    441443    for (i = 0; i < Nrows; i++, tblptr_start += Nx) {
    442444      char *tblptr = tblptr_start;
    … …  
    449451}
    450452
    451 int gfits_collect_table_gzp2_alt (FTable *table, TableField *field, char *raw, int row_start, int Nrows) {
    452 
    453   off_t i, j, k;
     453int gfits_collect_table_gzp2_alt (FTable *table, TableField *field, char *raw, unsigned long int row_start, unsigned long int Nrows) {
     454
     455  off_t j, k;
    454456
    455457  // we are copying NN rows into the column which starts at XX and has MM bytes per row
    … …  
    458460
    459461  for (k = 0; k < field->pixsize; k++) {
     462    unsigned long int i;
    460463    for (i = 0; i < Nrows; i++) {
    461       int row = row_start + i;
     464      unsigned long int row = row_start + i;
    462465      char *rawptr = &raw[i*field->Nvalues + k*Nrows*field->Nvalues];
    463466# ifdef BYTE_SWAP     
    … …  
    468471      for (j = 0; j < field->Nvalues; j++, tblptr += field->pixsize, rawptr++) {
    469472        *rawptr = *tblptr;
    470         myAssert (rawptr - raw < Nrows*field->Nvalues*field->pixsize, "oops");
     473        myAssert (rawptr >= raw, "oops");
     474        myAssert ((unsigned long int)(rawptr - raw) < Nrows*field->Nvalues*field->pixsize, "oops");
    471475        myAssert (tblptr - table->buffer < table->header->Naxis[0]*table->header->Naxis[1], "oops");
    472476      }
Note: See TracChangeset for help on using the changeset viewer.