IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38422


Ignore:
Timestamp:
Jun 7, 2015, 11:40:02 AM (11 years ago)
Author:
eugene
Message:

fix the swapping for dvo compressed I/O

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ohana.20150429/src/libdvo/src/dvo_catalog_split.c

    r38419 r38422  
    1212  if (fullWrite && catcompress) return TRUE;  // we want to compress, so do not swap
    1313  return FALSE;
     14}
     15
     16int byteswap_varlength_ftable (Header *header, FTable *table) {
     17  int i, Nfields;
     18  if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE;
     19  for (i = 0; i < Nfields; i++) {
     20    if (!gfits_byteswap_varlength_column (table, i+1)) return FALSE;
     21  }
     22  return TRUE;
    1423}
    1524
    … …  
    124133    fprintf (stderr, "compress %s\n", catalog->filename);
    125134    char *compressMode = dvo_catalog_compress_string (catalog->catcompress);
    126     if (!gfits_compress_table (ftable, &cmptable, 0, compressMode)) {
     135    // I should test how Ntile affects fpack/funpack and speed
     136    if (!gfits_compress_table (ftable, &cmptable, 1000, compressMode)) {
    127137      fprintf (stderr, "compression failure\n");
    128138      return (FALSE);
     139    }
     140    if (!byteswap_varlength_ftable (&cmpheader, &cmptable)) {
     141      fprintf (stderr, "failed to swap varlength column\n");
     142      return FALSE;
    129143    }
    130144    if (!gfits_modify (cmptable.header, "DVO_CMP", "%s", 1, compressMode)) {
    … …  
    140154  if (fullWrite) {
    141155    if (!gfits_fwrite_Theader (catalog->f, outtable->header)) {
    142       fprintf (stderr, "can't write table data\n");
     156      fprintf (stderr, "can't write table header\n");
    143157      return (FALSE);
    144158    }
    … …  
    149163  } else {
    150164    if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) {
    151       fprintf (stderr, "can't write table data\n");
     165      fprintf (stderr, "can't write table data (range)\n");
    152166      return (FALSE);
    153167    }
    … …  
    257271    rawtable.header = &rawheader;
    258272
    259     // NOTE: uncompress swaps the bytes into native order
     273    // NOTE: uncompress swaps the data bytes into native order, but needs the varlength columns native
     274    if (!byteswap_varlength_ftable (ftable->header, ftable)) {
     275      fprintf (stderr, "failed to swap varlength column\n");
     276      return FALSE;
     277    }
    260278    if (!gfits_uncompress_table (ftable, &rawtable)) {
    261279      if (VERBOSE) fprintf (stderr, "failed to uncompress table\n");
    … …  
    819837    int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress);
    820838
     839    // convert to external table format
    821840    if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) {
    822841      fprintf (stderr, "trouble converting format\n");
    … …  
    855874    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
    856875
     876    // if we are going to compress, we need to receive unswapped data --
     877    int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
     878
    857879    // convert to external table format
    858     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
     880    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
    859881      fprintf (stderr, "trouble converting format\n");
    860882      goto failure;
    … …  
    878900    }
    879901
     902    // if we are going to compress, we need to receive unswapped data --
     903    int swapFromNative = !output_is_compressed (start, Nrows, Nmissing_disk_new, catalog->catcompress);
     904
    880905    // convert to external table format
    881     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
     906    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
    882907      fprintf (stderr, "trouble converting format\n");
    883908      goto failure;
    … …  
    907932    // XXX check these for consistency...
    908933
     934    // if we are going to compress, we need to receive unswapped data --
     935    int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
     936
    909937    // convert to external table format
    910     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
     938    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative)) {
     939      fprintf (stderr, "trouble converting format\n");
     940      goto failure;
     941    }
    911942
    912943    // write out SecFilt table
    … …  
    931962    assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off);
    932963
     964    // if we are going to compress, we need to receive unswapped data --
     965    int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
     966
    933967    // convert to external table format
    934     if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
     968    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
    935969      fprintf (stderr, "trouble converting format\n");
    936970      goto failure;
    … …  
    958992    assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off);
    959993
     994    // if we are going to compress, we need to receive unswapped data --
     995    int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
     996
    960997    // convert to external table format
    961     if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
     998    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
    962999      fprintf (stderr, "trouble converting format\n");
    9631000      goto failure;
    … …  
    9851022    assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off);
    9861023
     1024    // if we are going to compress, we need to receive unswapped data --
     1025    int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
     1026
    9871027    // convert to external table format
    988     if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
     1028    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
    9891029      fprintf (stderr, "trouble converting format\n");
    9901030      goto failure;
    … …  
    10121052    assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off);
    10131053
     1054    // if we are going to compress, we need to receive unswapped data --
     1055    int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress);
     1056
    10141057    // convert to external table format
    1015     if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
     1058    if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) {
    10161059      fprintf (stderr, "trouble converting format\n");
    10171060      goto failure;
    … …  
    11521195    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
    11531196
    1154     // if we are going to compress, we need to receive unswapped data --
    1155     int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
    1156 
    11571197    // convert to external table format (note that the block above does not damage or free catalog.average)
    1158     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1198    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
    11591199      fprintf (stderr, "trouble converting format\n");
    11601200      goto failure;
    … …  
    11781218    }
    11791219
    1180     // if we are going to compress, we need to receive unswapped data --
    1181     int swapFromNative = !output_is_compressed (0, catalog->Nmissing, catalog->Nmissing, catalog->catcompress);
    1182 
    11831220    // convert to external table format
    1184     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
     1221    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
    11851222      fprintf (stderr, "trouble converting format\n");
    11861223      goto failure;
    … …  
    12101247    // XXX check these for consistency...
    12111248
    1212     // if we are going to compress, we need to receive unswapped data --
    1213     int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
    1214 
    12151249    // convert to external table format
    1216     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative);
     1250    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
     1251      fprintf (stderr, "trouble converting format\n");
     1252      goto failure;
     1253    }
    12171254
    12181255    // write out SecFilt table
    … …  
    12381275
    12391276    if (catalog[0].Nlensing) {
    1240       // if we are going to compress, we need to receive unswapped data --
    1241       int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
    1242 
    12431277      // convert to external table format (note that the block above does not damage or free catalog.average)
    1244       if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1278      if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
    12451279        fprintf (stderr, "trouble converting format\n");
    12461280        goto failure;
    … …  
    12701304
    12711305    if (catalog[0].Nlensobj) {
    1272       // if we are going to compress, we need to receive unswapped data --
    1273       int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
    1274 
    12751306      // convert to external table format (note that the block above does not damage or free catalog.average)
    1276       if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1307      if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
    12771308        fprintf (stderr, "trouble converting format\n");
    12781309        goto failure;
    … …  
    13021333
    13031334    if (catalog[0].Nstarpar) {
    1304       // if we are going to compress, we need to receive unswapped data --
    1305       int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
    1306 
    13071335      // convert to external table format (note that the block above does not damage or free catalog.average)
    1308       if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1336      if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
    13091337        fprintf (stderr, "trouble converting format\n");
    13101338        goto failure;
    … …  
    13341362
    13351363    if (catalog[0].Ngalphot) {
    1336       // if we are going to compress, we need to receive unswapped data --
    1337       int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress);
    1338 
    13391364      // convert to external table format (note that the block above does not damage or free catalog.average)
    1340       if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1365      if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
    13411366        fprintf (stderr, "trouble converting format\n");
    13421367        goto failure;
    … …  
    15661591
    15671592    // convert to external table format
    1568     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE);
     1593    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
     1594      fprintf (stderr, "trouble converting format\n");
     1595      goto failure;
     1596    }
    15691597
    15701598    // write out SecFilt table
Note: See TracChangeset for help on using the changeset viewer.