IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38446


Ignore:
Timestamp:
Jun 12, 2015, 9:43:43 PM (11 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20150419/Ohana/src/dvomerge
Files:
29 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/Makefile

    r38285 r38446  
    166166$(BIN)/dvoutils.$(ARCH) : $(DVOUTILS)
    167167
    168 INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify
     168INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client dvoverify dvoutils
    169169
    170170# dependancy rules for binary code #########################

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/include/dvomerge.h

    r38285 r38446  
    4242int    REPLACE_TYCHO;
    4343int    FORCE_MERGE;
     44int    MAX_CLIENTS;
    4445char  *UPDATE_CATFORMAT;
     46char  *UPDATE_CATCOMPRESS;
    4547
    4648int    MATCHED_TABLES;

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadCatalog.c

    r37807 r38446  
    88
    99    // always load all of the data (if any exists)
    10     catalog[0].catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_LENSING | LOAD_LENSOBJ | LOAD_STARPAR;
     10    catalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
    1111
    1212    catalog[0].catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/LoadImages.c

    r29938 r38446  
    3434  }
    3535
    36   image = gfits_table_get_Image (&db->ftable, Nimage, &db->swapped);
     36  image = gfits_table_get_Image (&db->ftable, Nimage, &db->scaledValue, &db->nativeOrder);
    3737  if (!image) {
    3838    fprintf (stderr, "ERROR: failed to read images\n");

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/args.c

    r38285 r38446  
    4444  }
    4545
     46  /* limit the impact of a dvomerge -parallel */
     47  MAX_CLIENTS = 10;
     48  if ((N = get_argument (*argc, argv, "-max-clients"))) {
     49    remove_argument (N, argc, argv);
     50    MAX_CLIENTS = atoi(argv[N]);
     51    remove_argument (N, argc, argv);
     52  }
     53
    4654  /* use a different photcode file to define mean values */
    4755  if ((N = get_argument (*argc, argv, "-photcode-file"))) {
    … …  
    6270    remove_argument (N, argc, argv);
    6371    UPDATE_CATFORMAT = strcreate (argv[N]);
     72    remove_argument (N, argc, argv);
     73  }
     74  UPDATE_CATCOMPRESS = NULL;
     75  if ((N = get_argument (*argc, argv, "-update-catcompress"))) {
     76    remove_argument (N, argc, argv);
     77    UPDATE_CATCOMPRESS = strcreate (argv[N]);
    6478    remove_argument (N, argc, argv);
    6579  }
    … …  
    219233    remove_argument (N, argc, argv);
    220234  }
     235  UPDATE_CATCOMPRESS = NULL;
     236  if ((N = get_argument (*argc, argv, "-update-catcompress"))) {
     237    remove_argument (N, argc, argv);
     238    UPDATE_CATCOMPRESS = strcreate (argv[N]);
     239    remove_argument (N, argc, argv);
     240  }
    221241
    222242  /* replace measurement, don't duplicate */

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

    r37807 r38446  
    1616  short  *TgtCodes;
    1717 
    18   imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].swapped);
     18  imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].scaledValue, &src[0].nativeOrder);
    1919  if (!imagesSrc) {
    2020    fprintf (stderr, "ERROR: failed to read images from src\n");
    … …  
    2222  }
    2323
    24   imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].swapped);
     24  imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].scaledValue, &tgt[0].nativeOrder);
    2525  if (!imagesTgt) {
    2626    fprintf (stderr, "ERROR: failed to read images from tgt\n");
    … …  
    8383  }
    8484
    85   images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped);
     85  images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].scaledValue, &in[0].nativeOrder);
    8686  if (!images) {
    8787    fprintf (stderr, "ERROR: failed to read images\n");

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoconvert.c

    r33657 r38446  
    8282
    8383      // skip empty input catalogs
    84       if (!incatalog.Naves_disk) {
     84      if (!incatalog.Naverage_disk) {
    8585        dvo_catalog_unlock (&incatalog);
    8686        dvo_catalog_free (&incatalog);
    … …  
    9393    SkyListFree (inlist);
    9494
    95     outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
     95    outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
    9696    dvo_catalog_save (&outcatalog, VERBOSE);
    9797    dvo_catalog_unlock (&outcatalog);
    … …  
    122122  /*** load output/Images.dat ***/
    123123  sprintf (ImageCat, "%s/Images.dat", output);
     124  gfits_db_init (&outDB);
    124125  outDB.mode   = dvo_catalog_catmode (CATMODE);
    125126  outDB.format = dvo_catalog_catformat (CATFORMAT);
    … …  
    135136  /*** load input/Images.dat ***/
    136137  sprintf (ImageCat, "%s/Images.dat", input);
     138  gfits_db_init (&inDB);
    137139  // inDB.mode   = dvo_catalog_catmode (CATMODE);
    138140  // inDB.format = dvo_catalog_catformat (CATFORMAT);
    … …  
    149151
    150152  // convert the raw image table to Image type (byteswap if needed)
    151   images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped);
     153  images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.scaledValue, &inDB.nativeOrder);
    152154  if (!images) {
    153155    fprintf (stderr, "ERROR: failed to read images\n");

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue.c

    r37116 r38446  
    109109    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    110110    // skip empty input catalogs
    111     if (!incatalog.Naves_disk) {
     111    if (!incatalog.Naverage_disk) {
    112112        dvo_catalog_unlock (&incatalog);
    113113        dvo_catalog_free (&incatalog);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c

    r37116 r38446  
    4747    LoadCatalog (&incatalog, threadData->region, threadData->filename, "r", threadData->NsecfiltInput);
    4848    // skip empty input catalogs
    49     if (!incatalog.Naves_disk) {
     49    if (!incatalog.Naverage_disk) {
    5050      dvo_catalog_unlock (&incatalog);
    5151      dvo_catalog_free (&incatalog);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeCreate.c

    r37116 r38446  
    145145
    146146      // skip empty input catalogs
    147       if (!incatalog.Naves_disk) {
     147      if (!incatalog.Naverage_disk) {
    148148        dvo_catalog_unlock (&incatalog);
    149149        dvo_catalog_free (&incatalog);
    … …  
    166166
    167167      // skip empty input catalogs
    168       if (!incatalog.Naves_disk) {
     168      if (!incatalog.Naverage_disk) {
    169169        dvo_catalog_unlock (&incatalog);
    170170        dvo_catalog_free (&incatalog);
    … …  
    204204    /*** load output/Images.dat ***/
    205205    sprintf (ImageCat, "%s/Images.dat", output);
     206    gfits_db_init (&outDB);
    206207    outDB.mode   = dvo_catalog_catmode (CATMODE);
    207208    outDB.format = dvo_catalog_catformat (CATFORMAT);
    … …  
    218219    /*** load input1/Images.dat ***/
    219220    sprintf (ImageCat, "%s/Images.dat", input1);
     221    gfits_db_init (&in1DB);
    220222    in1DB.mode   = dvo_catalog_catmode (CATMODE);
    221223    in1DB.format = dvo_catalog_catformat (CATFORMAT);
    … …  
    237239    /*** load input2/Images.dat ***/
    238240    sprintf (ImageCat, "%s/Images.dat", input2);
     241    gfits_db_init (&in2DB);
    239242    in2DB.mode   = dvo_catalog_catmode (CATMODE);
    240243    in2DB.format = dvo_catalog_catformat (CATFORMAT);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeFromList.c

    r37116 r38446  
    113113
    114114    // skip empty input catalogs
    115     if (!incatalog.Naves_disk) {
     115    if (!incatalog.Naverage_disk) {
    116116        dvo_catalog_unlock (&incatalog);
    117117        dvo_catalog_free (&incatalog);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeImageIDs.c

    r36833 r38446  
    1010  /*** load input1/Images.dat ***/
    1111  sprintf (ImageCat, "%s/Images.dat", input);
     12  gfits_db_init (&inDB);
    1213  inDB.mode   = dvo_catalog_catmode (CATMODE);
    1314  inDB.format = dvo_catalog_catformat (CATFORMAT);
    14   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
     15  status      = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
    1516  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    1617
    … …  
    3738  /*** load output/Images.dat ***/
    3839  sprintf (ImageCat, "%s/Images.dat", output);
     40  gfits_db_init (&outDB);
    3941  outDB.mode   = dvo_catalog_catmode (CATMODE);
    4042  outDB.format = dvo_catalog_catformat (CATFORMAT);
    … …  
    9395  /*** load input1/Images.dat ***/
    9496  sprintf (ImageCat, "%s/Images.dat", input);
     97  gfits_db_init (&inDB);
    9598  inDB.mode   = dvo_catalog_catmode (CATMODE);
    9699  inDB.format = dvo_catalog_catformat (CATFORMAT);
    97   status       = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
     100  status      = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
    98101  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    99102
    … …  
    115118  /*** load output/Images.dat ***/
    116119  sprintf (ImageCat, "%s/Images.dat", output);
     120  gfits_db_init (&outDB);
    117121  outDB.mode   = dvo_catalog_catmode (CATMODE);
    118122  outDB.format = dvo_catalog_catformat (CATFORMAT);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

    r38285 r38446  
    145145
    146146    // skip empty input catalogs
    147     if (!incatalog.Naves_disk) {
     147    if (!incatalog.Naverage_disk) {
    148148        dvo_catalog_unlock (&incatalog);
    149149        dvo_catalog_free (&incatalog);
    … …  
    179179
    180180      // if no catalog already exists, use the input catalog to define the format or the specified format
    181       if (outcatalog.Naves_disk == 0) {
    182         if (UPDATE_CATFORMAT) {
    183           outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
    184         } else {
    185           outcatalog.catformat = incatalog.catformat;
    186         }
     181      if (UPDATE_CATFORMAT) {
     182        outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
     183      } else {
     184        outcatalog.catformat = incatalog.catformat;
     185      }
     186      if (UPDATE_CATCOMPRESS) {
     187        outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS);
     188      } else {
     189        outcatalog.catcompress = incatalog.catcompress;
    187190      }
    188191
    … …  
    230233  return TRUE;
    231234}
     235
     236int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput);
    232237
    233238// launch the dvomergeUpdate_client jobs to the parallel hosts
    … …  
    257262  }   
    258263
    259 # if (MACHINE_GROUPS)
    260264  int Ngroups;
    261   HostTableGroup *groups = HostTableGroups (table, &Ngroups);
     265  HostTableGroup *groups = HostTableGroupsMaxNumber (table, &Ngroups, MAX_CLIENTS);
    262266  // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines)
    263267
    … …  
    267271    dvomergeUpdate_parallel_group (&groups[i], absinput, absoutput, outsky);
    268272  }
    269 # else
    270   dvomergeUpdate_parallel_table (table, absinput, absoutput);
    271 # endif
    272 
    273273  return TRUE;
    274274}
    275275
    276 // launch the dvomergeUpdate_client jobs to the parallel hosts
    277 int dvomergeUpdate_parallel_table (HostTable *table, char *absinput, char *absoutput) {
     276int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput) {
    278277
    279278  int i;
    280   for (i = 0; i < table->Nhosts; i++) {
     279  for (i = 0; i < group->Nhosts; i++) {
    281280
    282281    // ensure that the paths are absolute path names
    283     char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
    284     free (table->hosts[i].pathname);
    285     table->hosts[i].pathname = tmppath;
     282    char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH);
     283    free (group->hosts[i][0].pathname);
     284    group->hosts[i][0].pathname = tmppath;
    286285
    287286    // options / arguments that can affect relastro_client -update-objects:
    288287    char command[DVO_MAX_PATH];
    289288    snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f",
    290               absinput, absoutput, table->hosts[i].hostID, table->hosts[i].pathname,
     289              absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname,
    291290              UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS
    292291      );
    … …  
    315314      int status = system (command);
    316315      if (status) {
    317         fprintf (stderr, "ERROR running photdbc_client\n");
    318         exit (2);
    319       }
    320     } else {
    321       // launch the job on the remote machine (no handshake)
    322       int errorInfo = 0;
    323       int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
    324       if (!pid) {
    325         if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
    326         exit (1);
    327       }
    328       table->hosts[i].pid = pid; // save for future reference
    329     }
    330   }
    331 
    332   if (PARALLEL_MANUAL) {
    333     fprintf (stderr, "run the photdbc_client commands above.  when these are done, hit return\n");
    334     getchar();
    335   }
    336   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
    337     int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
    338     if (!status) {
    339       fprintf (stderr, "error running one of the remote clients\n");
    340       return status;
    341     }
    342   }
    343 
    344   return TRUE;
    345 }     
    346 
    347 // launch the dvomergeUpdate_client jobs to the parallel hosts
    348 int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput, SkyTable *sky) {
    349 
    350   int i, j;
    351   for (i = 0; i < group->Nhosts; i++) {
    352 
    353     if (sky->Nregions < group->Nhosts) {
    354       // do any of the regions want this host?
    355       int wantThisHost = FALSE;
    356       for (j = 0; j < sky->Nregions; j++) {
    357         if (HostTableTestHost (&sky->regions[j], group->hosts[i][0].hostID)) {
    358           wantThisHost = TRUE;
    359           break;
    360         }
    361       }
    362       if (!wantThisHost) {
    363         // fprintf (stderr, "skip host %s\n", group->hosts[i][0].hostname);
    364         continue;
    365       }
    366     }
    367 
    368     // ensure that the paths are absolute path names
    369     char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH);
    370     free (group->hosts[i][0].pathname);
    371     group->hosts[i][0].pathname = tmppath;
    372 
    373     // options / arguments that can affect relastro_client -update-objects:
    374     char command[DVO_MAX_PATH];
    375     snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f",
    376               absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname,
    377               UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS
    378       );
    379 
    380     char tmpline[DVO_MAX_PATH];
    381     if (VERBOSE)             { snprintf (tmpline, DVO_MAX_PATH, "%s -v",                command); strcpy (command, tmpline); }
    382     if (VERIFY)              { snprintf (tmpline, DVO_MAX_PATH, "%s -verify",           command); strcpy (command, tmpline); }
    383     if (VERIFY_CATALOG_ONLY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify-catalogs",  command); strcpy (command, tmpline); }
    384     if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace",          command); strcpy (command, tmpline); }
    385     if (PARALLEL_INPUT)      { snprintf (tmpline, DVO_MAX_PATH, "%s -parallel-input",   command); strcpy (command, tmpline); }
    386     if (FORCE_MERGE)         { snprintf (tmpline, DVO_MAX_PATH, "%s -force-merge",      command); strcpy (command, tmpline); }
    387     if (MATCHED_TABLES)      { snprintf (tmpline, DVO_MAX_PATH, "%s -matched-tables",   command); strcpy (command, tmpline); }
    388     if (UPDATE_CATFORMAT)    { snprintf (tmpline, DVO_MAX_PATH, "%s -update-catformat %s", command, UPDATE_CATFORMAT); strcpy (command, tmpline); }
    389 
    390     // add some config variables:
    391     snprintf (tmpline, DVO_MAX_PATH, "%s -D CATMODE %s",    command, CATMODE);   strcpy (command, tmpline);
    392     snprintf (tmpline, DVO_MAX_PATH, "%s -D CATFORMAT %s",  command, CATFORMAT); strcpy (command, tmpline);
    393     snprintf (tmpline, DVO_MAX_PATH, "%s -D SKY_DEPTH %d",  command, SKY_DEPTH); strcpy (command, tmpline);
    394 
    395     fprintf (stderr, "command: %s\n", command);
    396 
    397     if (PARALLEL_MANUAL) continue;
    398 
    399     if (PARALLEL_SERIAL) {
    400       int status = system (command);
    401       if (status) {
    402         fprintf (stderr, "ERROR running photdbc_client\n");
     316        fprintf (stderr, "ERROR running dvomerge_client\n");
    403317        exit (2);
    404318      }
    … …  
    416330
    417331  if (PARALLEL_MANUAL) {
    418     fprintf (stderr, "run the photdbc_client commands above.  when these are done, hit return\n");
     332    fprintf (stderr, "run the dvomerge_client commands above.  when these are done, hit return\n");
    419333    getchar();
    420334  }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c

    r37116 r38446  
    4444    LoadCatalog (&incatalog, threadData->region, threadData->filename, "r", threadData->NsecfiltInput);
    4545    // skip empty input catalogs
    46     if (!incatalog.Naves_disk) {
     46    if (!incatalog.Naverage_disk) {
    4747      dvo_catalog_unlock (&incatalog);
    4848      dvo_catalog_free (&incatalog);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairCPT.c

    r37807 r38446  
    7676  }
    7777
    78   measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     78  measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    7979  myAssert(measure, "failed to convert ftable to measure data");
    8080
    … …  
    156156  gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t",      1,  FALSE);
    157157
     158  myAbort ("test for compression");
     159
    158160  /* convert internal to external format */
    159   if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL)) {
     161  if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL, TRUE)) {
    160162    myAbort("trouble converting format");
    161163  }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImageList.c

    r37807 r38446  
    4646  FILE *cpmFile = NULL;
    4747
    48   char catformat;
     48  DVOCatFormat catformat;
    4949
    5050  N = get_argument (argc, argv, "-delete-image-list");
    … …  
    246246        gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    247247
    248         measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     248        measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    249249        myAssert(measure, "failed to convert ftable to measure data");
    250250   
    … …  
    284284
    285285        // convert internal to external format
    286         if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) {
     286        if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) {
    287287          myAbort("trouble converting format");
    288288        }
    … …  
    467467  gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
    468468
     469  myAbort ("test for compression");
     470
    469471  /* convert internal to external format */
    470   if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) {
     472  if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) {
    471473    myAbort("trouble converting format");
    472474  }
    … …  
    529531
    530532    /* convert internal to external format */
    531     if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) {
     533    if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) {
    532534      myAbort("trouble converting format");
    533535    }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairDeleteImagesByExternID.c

    r38319 r38446  
    3838  char *imageFilename = NULL;
    3939
    40   char catformat;
     40  DVOCatFormat catformat;
    4141
    4242  N = get_argument (argc, argv, "-delete-images-by-extern-id");
    … …  
    279279        gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    280280
     281        myAbort ("fix compression");
     282
    281283        // this function can only handle PS1_V5 and later formats
    282284        // NOTE: FtableToMeasure (and equivalent) free the data associated with cpmFtable.
    283285        // We are left with the data in measure.
    284         measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     286        measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    285287        myAssert(measure, "failed to convert ftable to measure data");
    286288   
    … …  
    331333
    332334        // convert internal to external format
    333         if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) {
     335        if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) {
    334336          myAbort("trouble converting format");
    335337        }
    … …  
    493495
    494496  /* setup image table format and lock */
     497  gfits_db_init (&db);
    495498  db.mode    = dvo_catalog_catmode ("SPLIT");
    496499  db.format  = dvo_catalog_catformat ("PS1_V5");
    … …  
    658661  gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
    659662
     663  myAbort ("test for compression");
     664
    660665  /* convert internal to external format */
    661   if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) {
     666  if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) {
    662667    myAbort("trouble converting format");
    663668  }
    … …  
    727732
    728733    /* convert internal to external format */
    729     if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) {
     734    if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) {
    730735      myAbort("trouble converting format");
    731736    }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixCPT.c

    r37046 r38446  
    8686  Matrix matrix;
    8787
    88   char catformat;
     88  DVOCatFormat catformat;
    8989
    9090  cpmFtable.header = &cpmHeaderTBL;
    … …  
    120120  }
    121121
    122   measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     122  myAbort ("fix cpts");
     123  measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    123124  myAssert(measure, "failed to convert ftable to measure data");
    124125
    … …  
    127128
    128129  // convert internal to external format
    129   if (!MeasureToFtable (&cpmFtable, NULL, measure, Nmeasure, catformat)) {
     130  if (!MeasureToFtable (&cpmFtable, NULL, measure, Nmeasure, catformat, TRUE)) {
    130131    myAbort("trouble converting format");
    131132  }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixImages.c

    r37807 r38446  
    255255
    256256  /* setup image table format and lock */
     257  gfits_db_init (oldDB);
    257258  db.mode   = oldDB->mode;
    258259  db.format = oldDB->format;
    … …  
    280281  memset (&db.ftable.buffer[Nx*Nout], ' ', nbytes - Nx*Nout);
    281282
    282   db.swapped = TRUE; // internal representation
     283  db.nativeOrder = TRUE; // internal representation
     284  db.scaledValue = TRUE; // internal representation
    283285
    284286  gfits_modify (&db.header, "NIMAGES", OFF_T_FMT, 1,  Nout);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairFixTables.c

    r37807 r38446  
    103103
    104104      // convert internal to external format
    105       if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat)) {
     105      if (!MeasureToFtable (&cpmFtable, NULL, measureNew, NmeasureNew, catformat, TRUE)) {
    106106        myAbort("trouble converting format");
    107107      }
    … …  
    183183  gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows);
    184184
    185   measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     185  measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    186186  myAssert(measure, "failed to convert ftable to measure data");
    187187   
    … …  
    326326  gfits_scan (&cptHeaderPHU, "NSECFILT",     "%d",      1,  &Nsecfilt);
    327327
     328  myAbort ("test for compression");
     329
    328330  /* convert internal to external format */
    329   if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL)) {
     331  if (!AverageToFtable (&cptFtable, averageOut, NaverageOut, catformat, NULL, TRUE)) {
    330332    myAbort("trouble converting format");
    331333  }
    … …  
    388390
    389391    /* convert internal to external format */
    390     if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat)) {
     392    if (!SecFiltToFtable (&cpsFtable, secfilt, Nrows, catformat, TRUE)) {
    391393      myAbort("trouble converting format");
    392394    }

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImageVsMeasure.c

    r37046 r38446  
    128128      gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    129129
    130       measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     130      measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    131131      myAssert(measure, "failed to convert ftable to measure data");
    132132   

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvorepairImagesVsMeasures.c

    r37046 r38446  
    3232  FILE *cpmFile = NULL;
    3333
    34   char catformat;
     34  DVOCatFormat catformat;
    3535
    3636  N = get_argument (argc, argv, "-images-vs-measures");
    … …  
    134134      gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    135135
    136       measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat);
     136      measure = FtableToMeasure (&cpmFtable, NULL, &Nmeasure, &catformat, FALSE);
    137137      myAssert(measure, "failed to convert ftable to measure data");
    138138   

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvosecfilt_catalogs.c

    r38319 r38446  
    3939    // always load all of the data (if any exists)
    4040    catalog.Nsecfilt  = 0;
    41     catalog.catflags  = LOAD_SECF;  // XXX this will fail for MEF version
     41    catalog.catflags  = DVO_LOAD_SECFILT;  // XXX this will fail for MEF version
    4242    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    4343    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
    … …  
    4848    }
    4949
    50     if (catalog.Naves_disk == 0) {
     50    if (catalog.Naverage_disk == 0) {
    5151      dvo_catalog_unlock (&catalog);
    5252      dvo_catalog_free (&catalog);
    … …  
    6464
    6565    insec = catalog.secfilt;
    66     ALLOCATE (outsec, SecFilt, catalog.Naves_disk * Nsecfilt);
     66    ALLOCATE (outsec, SecFilt, catalog.Naverage_disk * Nsecfilt);
    6767
    68     for (k = 0; k < catalog.Naves_disk; k++) {
     68    for (k = 0; k < catalog.Naverage_disk; k++) {
    6969      for (j = 0; (j < catalog.Nsecfilt) && (j < Nsecfilt); j++) {
    7070        outsec[k*Nsecfilt + j] = insec[k*NsecInput + j];
    … …  
    7777    catalog.secfilt = outsec;
    7878    catalog.Nsecfilt = Nsecfilt;
    79     catalog.Nsecf_mem = Nsecfilt * catalog.Naves_disk;
    80     catalog.Nsecf_disk = Nsecfilt * catalog.Naves_disk;
     79    catalog.Nsecfilt_mem = Nsecfilt * catalog.Naverage_disk;
     80    catalog.Nsecfilt_disk = Nsecfilt * catalog.Naverage_disk;
    8181
    8282    dvo_catalog_save (&catalog, VERBOSE);

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_load_image_index.c

    r37807 r38446  
    8787
    8888  off_t Nimage;
    89   Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped);
     89  Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder);
    9090  if (!image) {
    9191    fprintf (stderr, "ERROR: failed to read images\n");

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoutils_uniq_images.c

    r37807 r38446  
    132132      }
    133133      IDlist[n] ++;
     134      if (VERBOSE && (IDlist[n] > 1)) {
     135        fprintf (stderr, "duplicate ext_id %d in file %s\n", extern_id[i], imFile);
     136      }
    134137    }
    135138

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/dvoverify_utils.c

    r37807 r38446  
    178178  catalog.catformat = DVO_FORMAT_UNDEF; // read-only,do not set the catformat
    179179  catalog.catmode   = DVO_MODE_UNDEF; // read-only, do not set the catmode
    180   catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_LENSING | LOAD_LENSOBJ;
     180  catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ;
    181181  catalog.Nsecfilt  = 0;
    182182 
    … …  
    187187  }
    188188
    189   // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
    190   if (catalog.Naves_disk == 0) {
     189  // Naverage_disk == 0 implies an empty catalog file, skip empty catalogs
     190  if (catalog.Naverage_disk == 0) {
    191191    dvo_catalog_unlock (&catalog);
    192192    dvo_catalog_free (&catalog);
    … …  
    467467
    468468  // load the iage database table
     469  gfits_db_init (&inDB);
    469470  status = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
    470471  if (!status) {
    … …  
    484485  }
    485486
    486   images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped);
     487  images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.scaledValue, &inDB.nativeOrder);
    487488  if (!images) {
    488489    fprintf (stderr, "ERROR: failed to read images from src\n");

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs.c

    r27435 r38446  
    5757  output[0].Naverage = Naverage;
    5858  output[0].Nmeasure = Nmeasure;
    59   output[0].Nsecf_mem = Naverage * Nsecfilt;
     59  output[0].Nsecfilt_mem = Naverage * Nsecfilt;
    6060 
    6161  if (VERBOSE) {

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_new.c

    r37814 r38446  
    115115  output[0].Nlensing = Nlensing;
    116116  output[0].Nstarpar = Nstarpar;
    117   output[0].Nsecf_mem = Naverage * NsecfiltOut;
     117  output[0].Nsecfilt_mem = Naverage * NsecfiltOut;
    118118
    119119  // If we are using dvomergeCreate to split an existing catalog, then the max objID in

  • branches/eam_branches/ipp-20150419/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r38319 r38446  
    112112
    113113  /* set up pointers for linked list of measure */
    114   if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) {
     114  if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeasure_disk)) {
    115115    // this version is only valid if we have done a full catalog load, and if the catalog
    116116    // is sorted while processed
    … …  
    517517  output[0].Nlensing = Nlens;
    518518  output[0].Nstarpar = Nstar;
    519   output[0].Nsecf_mem = Nave*NsecfiltOut;
     519  output[0].Nsecfilt_mem = Nave*NsecfiltOut;
    520520  if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas, Nlens: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n",  Nstars,  Nave,  Nmeas,  Nlens, Nmatch);
    521521
Note: See TracChangeset for help on using the changeset viewer.