IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 34116


Ignore:
Timestamp:
Jul 3, 2012, 10:52:18 AM (14 years ago)
Author:
eugene
Message:

update dvoverify to work on parallel dvo dbs

Location:
branches/eam_branches/ipp-20120627/Ohana/src/dvomerge
Files:
5 added
4 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20120627/Ohana/src/dvomerge/Makefile

    r33963 r34116  
    2020dvomerge_client : $(BIN)/dvomerge_client.$(ARCH)
    2121dvoconvert      : $(BIN)/dvoconvert.$(ARCH)
     22dvoverify       : $(BIN)/dvoverify.$(ARCH)
     23dvoverify_client : $(BIN)/dvoverify_client.$(ARCH)
    2224
    23 all: dvomerge dvomerge_client dvoconvert dvosecfilt
     25all: dvomerge dvomerge_client dvoconvert dvosecfilt dvoverify dvoverify_client
    2426
    2527#  $(SRC)/dvomergeContinue.$(ARCH).o
    … …  
    4951
    5052$(DVOMERGE)  : $(INC)/dvomerge.h
    51 
    5253$(BIN)/dvomerge.$(ARCH) : $(DVOMERGE)
    5354
    … …  
    7273
    7374$(DVOMERGE_CLIENT) : $(INC)/dvomerge.h
    74 
    7575$(BIN)/dvomerge_client.$(ARCH) : $(DVOMERGE_CLIENT)
    7676
    … …  
    8787
    8888$(DVOCONVERT)  : $(INC)/dvomerge.h
    89 
    9089$(BIN)/dvoconvert.$(ARCH) : $(DVOCONVERT)
    9190
    … …  
    9998
    10099$(DVOSECFILT)  : $(INC)/dvomerge.h
    101 
    102100$(BIN)/dvosecfilt.$(ARCH) : $(DVOSECFILT)
    103101
    … …  
    116114
    117115$(DVOREPAIR)  : $(INC)/dvomerge.h
    118 
    119116$(BIN)/dvorepair.$(ARCH) : $(DVOREPAIR)
    120117
    121118DVOVERIFY = \
    122 $(SRC)/dvoverify.$(ARCH).o
     119$(SRC)/dvoverify.$(ARCH).o \
     120$(SRC)/dvoverify_args.$(ARCH).o \
     121$(SRC)/dvoverify_catalogs.$(ARCH).o \
     122$(SRC)/dvoverify_utils.$(ARCH).o
    123123
    124 $(DVOVERIFY)  : $(INC)/dvomerge.h
    125 
     124$(DVOVERIFY)  : $(INC)/dvoverify.h
    126125$(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
    127126
    128 INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify
     127DVOVERIFY_CLIENT = \
     128$(SRC)/dvoverify_client.$(ARCH).o \
     129$(SRC)/dvoverify_args.$(ARCH).o \
     130$(SRC)/dvoverify_catalogs.$(ARCH).o \
     131$(SRC)/dvoverify_utils.$(ARCH).o
     132
     133$(DVOVERIFY_CLIENT) : $(INC)/dvoverify.h
     134$(BIN)/dvoverify_client.$(ARCH) : $(DVOVERIFY_CLIENT)
     135
     136INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify dvoverify_client
    129137
    130138# dependancy rules for binary code #########################

  • branches/eam_branches/ipp-20120627/Ohana/src/dvomerge/include/dvomerge.h

    r33963 r34116  
    1717# include <glob.h>
    1818
    19 # define DVO_MAX_PATH 1024
    20 
    2119int    PARALLEL;
    2220int    PARALLEL_MANUAL;
    … …  
    2523int    HOST_ID;
    2624char  *HOSTDIR;
    27 
    2825
    2926int    VERBOSE;

  • branches/eam_branches/ipp-20120627/Ohana/src/dvomerge/src/args.c

    r33963 r34116  
    5050  }
    5151
    52   // XXX for the moment, make this selection manual.  it needs to be automatic
    53   // based on the state of the SkyTable
    5452  PARALLEL = FALSE;
    5553  if ((N = get_argument (*argc, argv, "-parallel"))) {

  • branches/eam_branches/ipp-20120627/Ohana/src/dvomerge/src/dvoverify.c

    r33963 r34116  
    1 # include "dvomerge.h"
     1# include "dvoverify.h"
    22
    33/* things we can verify easily:
    … …  
    88*/
    99
    10 int VerifyTableFile (char *filename);
    11 int CheckCatalogIndexes (char *catdir, char *filename,  SkyRegion *region);
    12 
    13 # define DEBUG 0
    14 
    15 int VERBOSE = FALSE;
    16 int NNotSorted = 0;
    17 
    1810int main (int argc, char **argv) {
    1911
    20   char filename[1024];
     12  SkyTable *sky;
     13  SkyList *skylist;
    2114
    22   int N, Nbad;
    23   off_t i;
    24   SkyTable *insky;
    25   SkyList *inlist;
    26   SkyRegion UserPatch;
    27   // Catalog catalog;
     15  // check various options
     16  dvoverify_args (&argc, argv);
     17  CATDIR = argv[1];
    2818
    29   int CHECKSORTED;
    30 
    31   if ((N = get_argument (argc, argv, "-v"))) {
    32     VERBOSE = TRUE;
    33     remove_argument (N, &argc, argv);
    34   }
    35   if ((N = get_argument (argc, argv, "-verbose"))) {
    36     VERBOSE = TRUE;
    37     remove_argument (N, &argc, argv);
    38   }
    39   if ((N = get_argument (argc, argv, "-s"))) {
    40     CHECKSORTED = TRUE;
    41     remove_argument (N, &argc, argv);
    42   }
    43   if ((N = get_argument (argc, argv, "-sorted"))) {
    44     CHECKSORTED = TRUE;
    45     remove_argument (N, &argc, argv);
    46   }
    47 
    48   if ((N = get_argument (argc, argv, "-cpt"))) {
    49     remove_argument (N, &argc, argv);
    50     char *filename = strcreate (argv[N]);
    51     remove_argument (N, &argc, argv);
    52 
    53     int isBad = FALSE;
    54 
    55     if (!VerifyTableFile (filename)) {
    56       fprintf (stderr, "bad average table %s\n", filename);
    57       isBad = TRUE;
    58     }
    59 
    60     // change last 't' to 's':
    61     int Nlast;
    62     Nlast = strlen(filename) - 1;
    63     filename[Nlast] = 's';
    64     if (!VerifyTableFile (filename)) {
    65       fprintf (stderr, "bad secfilt table %s\n", filename);
    66       isBad = TRUE;
    67     }
    68 
    69     // change last 't' to 's':
    70     Nlast = strlen(filename) - 1;
    71     filename[Nlast] = 'm';
    72     if (!VerifyTableFile (filename)) {
    73       fprintf (stderr, "bad measure table %s\n", filename);
    74       isBad = TRUE;
    75     }
    76 
    77     if (isBad) exit (1);
    78     exit (0);
    79   }
    80 
    81   // restrict to a portion of the sky
    82   UserPatch.Rmin = 0;
    83   UserPatch.Rmax = 360;
    84   UserPatch.Dmin = -90;
    85   UserPatch.Dmax = +90;
    86   if ((N = get_argument (argc, argv, "-region"))) {
    87     remove_argument (N, &argc, argv);
    88     UserPatch.Rmin = atof (argv[N]);
    89     remove_argument (N, &argc, argv);
    90     UserPatch.Rmax = atof (argv[N]);
    91     remove_argument (N, &argc, argv);
    92     UserPatch.Dmin = atof (argv[N]);
    93     remove_argument (N, &argc, argv);
    94     UserPatch.Dmax = atof (argv[N]);
    95     remove_argument (N, &argc, argv);
    96   }
    97 
    98   if (argc != 2) {
    99     fprintf (stderr, "USAGE: dvoverify (catdir) [-region Rmin Rmax Dmin Dmax] [-v] [-s]\n\n");
    100     fprintf (stderr, "  -v : VERBOSE\n");
    101     fprintf (stderr, "  -s : checks if sorted, return error if not\n");
    102     fprintf (stderr, "  (catdir) : database of interest\n");
    103     exit (2);
    104   }
    105 
    106   char *catdir = argv[1];
    107 
    108   Nbad = 0;
     19  int Nbad = 0;
    10920
    11021  // XXX make this step optional
    111   if (1) {
     22  if (CHECK_TOPLEVEL) {
     23    char filename[DVO_MAX_PATH];
     24
    11225    // check the photcode table
    113     sprintf (filename, "%s/Photcodes.dat", catdir);
     26    sprintf (filename, "%s/Photcodes.dat", CATDIR);
    11427    if (!VerifyTableFile (filename)) {
    11528      Nbad ++;
    … …  
    11730
    11831    // check the skytable
    119     char *skyfile = SkyTableFilename (catdir);
     32    char *skyfile = SkyTableFilename (CATDIR);
    12033    if (!VerifyTableFile (skyfile)) {
    12134      Nbad ++;
    … …  
    12336
    12437    // check the image table
    125     sprintf (filename, "%s/Images.dat", catdir);
     38    sprintf (filename, "%s/Images.dat", CATDIR);
    12639    if (!VerifyTableFile (filename)) {
    12740      Nbad ++;
    … …  
    13043
    13144  // load the sky table for the existing database
    132   insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
    133   myAssert(insky, "can't read SkyTable");
    134   SkyTableSetFilenames (insky, catdir, "cpt");
    135   inlist = SkyListByPatch (insky, -1, &UserPatch);
     45  sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     46  myAssert(sky, "can't read SkyTable");
     47  SkyTableSetFilenames (sky, CATDIR, "cpt");
     48  skylist = SkyListByPatch (sky, -1, &UserPatch);
    13649 
    137   // loop over all catalogs, save to output catalogs
    138   for (i = 0; i < inlist[0].Nregions; i++) {
    139     if (!inlist[0].regions[i][0].table) continue;
    140     if (i % 1000 == 0) fprintf (stderr, ".");
    141 
    142     // sprintf (filename, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name);
    143     if (!VerifyTableFile (inlist[0].filename[i])) {
    144       Nbad ++;
    145     }
    146 
    147     sprintf (filename, "%s/%s.cps", catdir, inlist[0].regions[i][0].name);
    148     if (!VerifyTableFile (filename)) {
    149       Nbad ++;
    150     }
    151 
    152     sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
    153     if (!VerifyTableFile (filename)) {
    154       Nbad ++;
    155     }
    156 
    157     if (!CheckCatalogIndexes(catdir, inlist[0].filename[i], inlist[0].regions[i])){
    158       Nbad ++;
    159     }
    160 
    161     // exit immediately if any file are unsorted and we require sorted tables
    162     if (CHECKSORTED && NNotSorted) {
    163       fprintf (stderr, "ERROR: files are not sorted\n");
    164       exit (1);
    165     }
    166   }
     50  dvoverify_catalogs (skylist, &Nbad);
    16751
    16852  if (Nbad > 0) {
    … …  
    17862}
    17963
    180 // is this file a consistent FITS file?
    181 int VerifyTableFile (char *filename) {
    182 
    183   int status, Next;
    184   off_t Nbytes;
    185   Header header;
    186 
    187   struct stat fileStats;
    188   FILE *file;
    189 
    190   // does the file exist?
    191   status = stat (filename, &fileStats);
    192   if (status) {
    193     // some error accessing the file.  there is only one acceptable error: file not found
    194     switch (errno) {
    195       case ENOENT:
    196         if (DEBUG) fprintf (stderr, "file does not exist, skipping %s\n", filename);
    197         return TRUE;
    198       case ENOMEM:
    199         fprintf (stderr, "Out of memory: %s\n", filename);
    200         return TRUE;
    201       case EACCES:
    202         fprintf (stderr, "Permission error on %s\n", filename);
    203         return FALSE;
    204       case EFAULT:
    205         fprintf (stderr, "Bad address: %s\n", filename);
    206         return FALSE;
    207       case ELOOP:
    208         fprintf (stderr, "Too many symbolic links encountered while traversing the path: %s\n", filename);
    209         return FALSE;
    210       case ENAMETOOLONG:
    211         fprintf (stderr, "File name too long: %s\n", filename);
    212         return FALSE;
    213       case ENOTDIR:
    214         fprintf (stderr, "A component of the path is not a directory: %s\n", filename);
    215         return FALSE;
    216       case EOVERFLOW:
    217         fprintf (stderr, "file too large for program version: %s\n", filename);
    218         return FALSE;
    219       default:
    220         fprintf (stderr, "unknown error: %s\n", filename);
    221         return FALSE;
    222     }
    223   }
    224 
    225   // does it have any data?
    226   if (fileStats.st_size == 0) {
    227     fprintf (stderr, "file is empty: %s\n", filename);
    228     return FALSE;
    229   }
    230 
    231   // can we open it?
    232   file = fopen(filename, "r");
    233   if (!file) {
    234     fprintf (stderr, "unable to open valid file: %s\n", filename);
    235     return FALSE;
    236   }
    237 
    238   // scan all extentions
    239   Nbytes = 0;
    240   Next = -1;
    241   if (DEBUG) fprintf (stderr, "sizes: ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nbytes, fileStats.st_size);
    242   while (Nbytes < fileStats.st_size) {
    243 
    244     // Check on the PHU
    245     if (!gfits_fread_header (file, &header)) {
    246       if (Next == -1) {
    247         fprintf (stderr, "unable to read PHU header for %s\n", filename);
    248       } else {
    249         fprintf (stderr, "unable to read header for %s, extension %d (or file has excess bytes)\n", filename, Next);
    250       }
    251       fclose(file);
    252       return (FALSE);
    253     }
    254 
    255     // move to TBL header
    256     Nbytes += header.datasize + gfits_data_size (&header);
    257     if (DEBUG) fprintf (stderr, "sizes: ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nbytes, fileStats.st_size);
    258     if (Nbytes > fileStats.st_size) {
    259       fprintf (stderr, "file is short ("OFF_T_FMT" vs "OFF_T_FMT"): %s\n", Nbytes, fileStats.st_size, filename);
    260       gfits_free_header(&header);
    261       fclose (file);
    262       return FALSE;
    263     }
    264     gfits_free_header(&header);
    265 
    266     status = fseeko (file, Nbytes, SEEK_SET);
    267     if (status) {
    268       switch (errno) {
    269         case EBADF:
    270           fprintf (stderr, "something wrong with file handle: %s\n", filename);
    271           fclose (file);
    272           return FALSE;
    273         case EINVAL:
    274           fprintf (stderr, "invalid offset: %s\n", filename);
    275           fclose (file);
    276           return FALSE;
    277         default:
    278           fprintf (stderr, "other error in fseeko: %s\n", filename);
    279           fclose (file);
    280           return FALSE;
    281       }
    282     }
    283     Next ++;
    284   }
    285   if (DEBUG) fprintf (stderr, "file is good: %s\n", filename);
    286   fclose (file);
    287   return TRUE;
    288 }
    289 
    290 // CheckCatalogIndexes(catdir, inlist[0].regions[i][0].name);
    291 
    292 int CheckCatalogIndexes (char *catdir, char *filename,  SkyRegion *region) {
    293 
    294   Catalog catalog;
    295   int i, j, m, status;
    296 
    297   status = TRUE;
    298 
    299   // set the parameters which guide catalog open/load/create
    300   catalog.filename  = filename;
    301   catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
    302   catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
    303   catalog.catflags  = LOAD_AVES | LOAD_MEAS;
    304   catalog.Nsecfilt  = 0;
    305  
    306   // an error exit status here is a significant error (disk I/O or file access)
    307   if (!dvo_catalog_open (&catalog, region, VERBOSE, "r")) {
    308     fprintf (stderr, "ERROR: failure to open catalog file %s\n", catalog.filename);
    309     return FALSE;
    310   }
    311 
    312   // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
    313   if (catalog.Naves_disk == 0) {
    314     dvo_catalog_unlock (&catalog);
    315     dvo_catalog_free (&catalog);
    316     return TRUE;
    317   }
    318 
    319   // if the table is SORTED, then the following can be checked
    320   // check the following:
    321   // measure[j].averef -> average[averef]
    322   // measure[j].objID = average[averef].objID
    323   // measure[j].catID = average[averef].catID
    324   // measure[j].measureOffset < Nmeasure
    325   // \sum average[].Nmeasure = Nmeasure
    326 
    327   // if the table is NOT SORTED, we have a subset of checks we can make
    328   if (!catalog.sorted) {
    329     fprintf (stderr, "!");
    330     dvo_catalog_unlock (&catalog);
    331     dvo_catalog_free (&catalog);
    332     NNotSorted++;
    333     if (VERBOSE) fprintf (stderr, "file is not sorted: %s\n", filename);
    334     return TRUE;
    335   }
    336 
    337   int NmeasureTotal = 0;
    338   int measureOffsetOK = TRUE;
    339   for (i = 0; i < catalog.Naverage; i++) {
    340     NmeasureTotal += catalog.average[i].Nmeasure;
    341     if (VERBOSE && !(NmeasureTotal <= catalog.Nmeasure)) {
    342       fprintf (stderr, "NmeasureTotal > catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure);
    343     }
    344     measureOffsetOK &= (catalog.average[i].measureOffset < catalog.Nmeasure);
    345     if (VERBOSE && !(catalog.average[i].measureOffset < catalog.Nmeasure)) {
    346       fprintf (stderr, "measureOffset >= catalog.Nmeasure: %d %d %d\n", i, catalog.average[i].measureOffset, (int) catalog.Nmeasure);
    347     }
    348     measureOffsetOK &= (catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure);
    349     if (VERBOSE && !(catalog.average[i].measureOffset + catalog.average[i].Nmeasure <= catalog.Nmeasure)) {
    350       fprintf (stderr, "measureOffset + Nmeasure > catalog.Nmeasure : %d %d %d\n", i, catalog.average[i].Nmeasure, (int) catalog.Nmeasure);
    351     }
    352   }
    353 
    354   if (!measureOffsetOK) {
    355     fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog.filename);
    356     status = FALSE;
    357   }
    358 
    359   if (NmeasureTotal != catalog.Nmeasure) {
    360     fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog.filename);
    361     status = FALSE;
    362   }
    363 
    364   if (!status) {
    365     dvo_catalog_unlock (&catalog);
    366     dvo_catalog_free (&catalog);
    367     return (status);
    368   }
    369 
    370   int objIDsOK = TRUE;
    371   int catIDsOK = TRUE;
    372   int averefOK = TRUE;
    373 
    374   for (i = 0; i < catalog.Naverage; i++) {
    375     m = catalog.average[i].measureOffset;
    376     for (j = 0; j < catalog.average[i].Nmeasure; j++) {
    377       objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID);
    378       catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID);
    379       averefOK &= (catalog.measure[m+j].averef == i);
    380     }
    381   }
    382    
    383   if (!objIDsOK) {
    384     fprintf (stderr, "ERROR: catalog %s has invalid obj IDs\n", catalog.filename);
    385     status = FALSE;
    386   }
    387   if (!catIDsOK) {
    388     fprintf (stderr, "ERROR: catalog %s has invalid cat IDs\n", catalog.filename);
    389     status = FALSE;
    390   }
    391   if (!averefOK) {
    392     fprintf (stderr, "ERROR: catalog %s has invalid averef values\n", catalog.filename);
    393     status = FALSE;
    394   }
    395 
    396 //  for (i = 0; i < catalog.Naverage; i++) {
    397 //    m = catalog.average[i].measureOffset;
    398 //    for (j = 0; i < catalog.Nmeasure; i++) {
    399 //      objIDsOK &= (catalog.average[i].objID == catalog.measure[m+j].objID);
    400 //      catIDsOK &= (catalog.average[i].catID == catalog.measure[m+j].catID);
    401 //      averefOK &= (catalog.measure[m+j].averef = i);
    402 //    }
    403 //  }
    404 
    405   dvo_catalog_unlock (&catalog);
    406   dvo_catalog_free (&catalog);
    407 
    408   return status;
    409 }
    410 
    411 // gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
    412 // gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows);
    413    
    414  
Note: See TracChangeset for help on using the changeset viewer.