IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 5, 2010, 10:04:42 PM (16 years ago)
Author:
eugene
Message:

handle errors in dvo I/O functions; better support for DIS images; merge in high-speed mods from Niall; systematic errors in astrometry (somewhat hackish); add fits I/O option to succeed reading a short file (padding as needed); various kapa improvements (frames consistent in X, PS, PNG; fix tick label signif digits; add function for image overlays; better thick box lines; PNG of the image); bDrawBuffer mods to drop static buffer and refer to passed through buffer; ResizeByImage; DefineSectionByImage; buutos to do png & jpeg; add program roc for raid over cluster; support for mosaic photcodes in dvo image plot; densify; section name [-image x y] : width based on current image; resize -by-image; threaded addstar -resort; threaded dvomerge; dvorepair; dvoverify; dvomerge continue; dvomerge from list (of regions)

Location:
trunk/Ohana/src/dvomerge
Files:
12 edited
19 copied

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvomerge/Makefile

    r29001 r29938  
    2222all: dvomerge dvoconvert dvosecfilt
    2323
     24#  $(SRC)/dvomergeContinue.$(ARCH).o
     25
    2426DVOMERGE = \
    2527$(SRC)/dvomerge.$(ARCH).o \
    2628$(SRC)/dvomergeUpdate.$(ARCH).o \
    2729$(SRC)/dvomergeCreate.$(ARCH).o \
     30$(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
     31$(SRC)/dvomergeFromList.$(ARCH).o \
     32$(SRC)/dvomergeImageIDs.$(ARCH).o \
    2833$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
    2934$(SRC)/SetSignals.$(ARCH).o \
    … …  
    7075$(BIN)/dvosecfilt.$(ARCH) : $(DVOSECFILT)
    7176
    72 INSTALL = dvomerge dvoconvert dvosecfilt
     77DVOREPAIR = \
     78$(SRC)/dvorepair.$(ARCH).o \
     79$(SRC)/dvorepairFixCPT.$(ARCH).o \
     80$(SRC)/dvorepairImagesVsMeasures.$(ARCH).o \
     81$(SRC)/dvorepairDeleteImageList.$(ARCH).o \
     82$(SRC)/dvorepairFixImages.$(ARCH).o \
     83$(SRC)/psps_ids.$(ARCH).o \
     84$(SRC)/LoadImages.$(ARCH).o \
     85$(SRC)/ReadDeleteList.$(ARCH).o \
     86$(SRC)/match_image.$(ARCH).o \
     87$(SRC)/help.$(ARCH).o \
     88$(SRC)/ImageOps.$(ARCH).o
     89
     90$(DVOREPAIR)  : $(INC)/dvomerge.h
     91
     92$(BIN)/dvorepair.$(ARCH) : $(DVOREPAIR)
     93
     94DVOVERIFY = \
     95$(SRC)/dvoverify.$(ARCH).o
     96
     97$(DVOVERIFY)  : $(INC)/dvomerge.h
     98
     99$(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
     100
     101INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify
    73102
    74103# dependancy rules for binary code #########################

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r29001 r29938  
    1717# include <glob.h>
    1818
     19# define myAbort(MSG) { fprintf (stderr, "%s\n", MSG); abort(); }
     20# define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } }
     21
    1922int    VERBOSE;
    2023char   CATDIR[256];
    … …  
    2528float  RADIUS;
    2629int    SKY_DEPTH;
     30int    NTHREADS;
    2731char   *ALTERNATE_PHOTCODE_FILE;
    2832
    … …  
    6569SkyList   *SkyTablePopulatedList_old  PROTO((SkyTable *sky, off_t Ns, off_t Ne));
    6670
    67 int        LoadCatalog            PROTO((Catalog *catalog, SkyRegion *region, char *filename, char *mode));
     71int        LoadCatalog            PROTO((Catalog *catalog, SkyRegion *region, char *filename, char *mode, int Nsecfilt));
    6872
    6973int        merge_catalogs_new     PROTO((SkyRegion *region, Catalog *output, Catalog *input, int *secflitMap));
    … …  
    8690off_t      dvo_map_image_ID       PROTO((IDmapType *IDmap, off_t oldID));
    8791int        dvo_update_image_IDs   PROTO((IDmapType *IDmap, Catalog *catalog));
     92
     93// dvorepair prototypes
     94Image     *LoadImages             PROTO((FITS_DB *db, char *filename, off_t *Nimage));
     95Image     *MatchImage             PROTO((Image *image, off_t Nimage, unsigned int time, short int source, unsigned int imageID));
     96off_t      match_image            PROTO((Image *image, off_t Nimage, unsigned int T, short int S));
     97
     98int        SaveImages             PROTO((FITS_DB *oldDB, char *filename, Image *imagesOut, off_t Nout));
     99
     100int        dvorepairFixCPT        PROTO((int argc, char **argv));
     101int        dvorepairImagesVsMeasures PROTO((int argc, char **argv));
     102int        dvorepairDeleteImageList PROTO((int argc, char **argv));
     103int        dvorepairFixImages     PROTO((int argc, char **argv));
     104
     105int       *ReadDeleteList         PROTO((char *filename, int *nindex));
     106int        RepairTableCPT         PROTO((char *cptFilenameSrc, char *cptFilenameTgt, char *cpsFilenameSrc, char *cpsFilenameTgt, Measure *measure, off_t Nmeasure, Image *image, off_t Nimage, char catformat));
     107void       dvorepair_help         PROTO((int argc, char **argv));
     108
     109off_t      getTgtIndex            PROTO((e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt));
     110void       SortTgtByTimes         PROTO((e_time *S, off_t *I, short *C, off_t N));
     111int        dvo_image_match_dbs    PROTO((IDmapType *IDmap, FITS_DB *tgt, FITS_DB *src));
     112int        dvomergeImagesGetMap   PROTO((IDmapType *IDmap, char *input, char *output));
     113int        dvomergeFromList       PROTO((int argc, char **argv));
     114int        dvomergeUpdate_threaded PROTO((int argc, char **argv));

  • trunk/Ohana/src/dvomerge/src/LoadCatalog.c

    r28329 r29938  
    11# include "dvomerge.h"
    22
    3 int LoadCatalog (Catalog *catalog, SkyRegion *region, char *filename, char *mode) {
     3int LoadCatalog (Catalog *catalog, SkyRegion *region, char *filename, char *mode, int Nsecfilt) {
    44
    55    // set the parameters which guide catalog open/load/create
    66    catalog[0].filename  = filename;
    7     catalog[0].Nsecfilt  = GetPhotcodeNsecfilt ();
     7    catalog[0].Nsecfilt  = Nsecfilt;
    88
    99    // always load all of the data (if any exists)

  • trunk/Ohana/src/dvomerge/src/args.c

    r29001 r29938  
    1515    remove_argument (N, argc, argv);
    1616    ALTERNATE_PHOTCODE_FILE = strdup(argv[N]);
     17    remove_argument (N, argc, argv);
     18  }
     19
     20  NTHREADS = 0;
     21  if ((N = get_argument (*argc, argv, "-threads"))) {
     22    remove_argument (N, argc, argv);
     23    NTHREADS = MAX(0, atoi(argv[N]));
    1724    remove_argument (N, argc, argv);
    1825  }
    … …  
    3643  }
    3744
    38   if ((*argc != 6) && (*argc != 4)) dvomerge_usage();
     45  if ((*argc < 4) || (*argc > 6)) dvomerge_usage();
    3946  return TRUE;
    4047}

  • trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

    r29001 r29938  
    22
    33void sort_IDmap (IDmapType *IDmap);
     4
     5off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
     6
     7  // use bisection to find the starting entry by time
     8
     9  off_t Nlo, Nhi, N;
     10
     11  // find the last TGT before start
     12  Nlo = 0; Nhi = NimagesTgt;
     13  while (Nhi - Nlo > 10) {
     14    N = 0.5*(Nlo + Nhi);
     15    if (TgtTimes[N] < start) {
     16      Nlo = MAX(N, 0);
     17    } else {
     18      Nhi = MIN(N + 1, NimagesTgt);
     19    }
     20  }
     21
     22  // check for the matched mosaic starting from Nlo
     23  // we may have to go much beyond Nlo since stop is not sorted
     24  // can we use a sorted version of stop to check when we are beyond the valid range??
     25  for (N = Nlo; N < NimagesTgt; N++) {
     26    if (TgtTimes[N] < start) continue;
     27    if (TgtTimes[N] > stop) return (-1);
     28    if (TgtCodes[N] != photcode) continue;
     29    return (TgtIndex[N]);
     30  }
     31  return (-1);
     32}
     33
     34// sort two times vectors and an index by first time vector
     35void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
     36
     37# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
     38  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
     39  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
     40  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
     41}
     42# define COMPARE(A,B)(S[A] < S[B])
     43
     44  OHANA_SORT (N, COMPARE, SWAPFUNC);
     45
     46# undef SWAPFUNC
     47# undef COMPARE
     48
     49}
     50
     51// we have two tables; 'tgt' contains some exposures from 'src' : find them and match a map
     52int dvo_image_match_dbs (IDmapType *IDmap, FITS_DB *tgt, FITS_DB *src) {
     53
     54  Image *imagesSrc, *imagesTgt;
     55  off_t NimagesSrc, NimagesTgt;
     56  off_t iSrc, iTgt;
     57  off_t  *TgtIndex;
     58  e_time *TgtTimes;
     59  short  *TgtCodes;
     60 
     61  imagesSrc = gfits_table_get_Image (&src[0].ftable, &NimagesSrc, &src[0].swapped);
     62  if (!imagesSrc) {
     63    fprintf (stderr, "ERROR: failed to read images from src\n");
     64    exit (2);
     65  }
     66
     67  imagesTgt = gfits_table_get_Image (&tgt[0].ftable, &NimagesTgt, &tgt[0].swapped);
     68  if (!imagesTgt) {
     69    fprintf (stderr, "ERROR: failed to read images from tgt\n");
     70    exit (2);
     71  }
     72
     73  ALLOCATE (IDmap->old, off_t, NimagesSrc);
     74  ALLOCATE (IDmap->new, off_t, NimagesSrc);
     75  IDmap->Nmap = NimagesSrc;
     76
     77  // match by time and photcode
     78  ALLOCATE (TgtIndex, off_t,  NimagesTgt);
     79  ALLOCATE (TgtTimes, e_time, NimagesTgt);
     80  ALLOCATE (TgtCodes, short,  NimagesTgt);
     81
     82  // save the Index, Times, Codes for all TGT images
     83  for (iTgt = 0; iTgt < NimagesTgt; iTgt++) {
     84    TgtIndex[iTgt] = iTgt;
     85    TgtTimes[iTgt] = imagesTgt[iTgt].tzero;
     86    TgtCodes[iTgt] = imagesTgt[iTgt].photcode;
     87  }
     88
     89  // sort the index, start, and stop by the start times:
     90  SortTgtByTimes (TgtTimes, TgtIndex, TgtCodes, NimagesTgt);
     91
     92  for (iSrc = 0; iSrc < NimagesSrc; iSrc++) {
     93    iTgt = getTgtIndex (imagesSrc[iSrc].tzero, imagesSrc[iSrc].tzero + (int) imagesSrc[iSrc].exptime, imagesSrc[iSrc].photcode, TgtIndex, TgtTimes, TgtCodes, NimagesTgt);
     94    if (iTgt < 0) Shutdown ("failure to match images: %s\n", imagesSrc[iSrc].name);
     95    IDmap[0].old[iSrc] = imagesSrc[iSrc].imageID;
     96    IDmap[0].new[iSrc] = imagesTgt[iTgt].imageID;
     97  }
     98
     99  FREE(TgtIndex);
     100  FREE(TgtTimes);
     101  FREE(TgtCodes);
     102
     103  // sort IDmap->old,new on the basis of IDmap->old:
     104  sort_IDmap (IDmap);
     105
     106  return TRUE;
     107}
    4108
    5109// merge db2 into db1

  • trunk/Ohana/src/dvomerge/src/dvoconvert.c

    r29001 r29938  
    66  char filename[256], *input, *output;
    77
    8   int depth;
     8  int depth, Nsecfilt;
    99  off_t i, j, Ns, Ne;
    1010  SkyTable *outsky, *insky;
    … …  
    3232      exit (1);
    3333    }
     34    Nsecfilt = GetPhotcodeNsecfilt();
     35
    3436    // save the photcodes in the output catdir
    3537    sprintf (filename, "%s/Photcodes.dat", output);
    … …  
    6769
    6870    // load / create output catalog
    69     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
     71    LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", Nsecfilt);
    7072
    7173    // combine only tables at equal or larger depth
    … …  
    7779
    7880      // load input catalog
    79       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     81      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", Nsecfilt);
    8082
    8183      // skip empty input catalogs

  • trunk/Ohana/src/dvomerge/src/dvomerge.c

    r28327 r29938  
    88  dvomerge_args (&argc, argv);
    99
    10   // XXX require both inputs to be sorted?
    11 
    12   if (argc == 6) dvomergeCreate (argc, argv);
    13   if (argc == 4) dvomergeUpdate (argc, argv);
     10  if (argc == 6) {
     11    if (!strcasecmp (argv[2], "and")) {
     12      dvomergeCreate (argc, argv);
     13    } else {
     14      dvomergeFromList (argc, argv);
     15    }
     16  }
     17  if ((argc == 4) || (argc == 5)) {
     18    if (NTHREADS) {
     19      dvomergeUpdate_threaded (argc, argv);
     20    } else {
     21      dvomergeUpdate (argc, argv);
     22    }
     23  }
    1424  dvomerge_usage();
    1525  exit (2); // cannot reach here.
    … …  
    1828/* we have two possible modes of operation:
    1929
    20    Create : dvomerge (in1) and (in2) to (out) -- create a new db from two input dbs
    21    Update : dvomerge (in) into (out)          -- merge a new db into an existing db
     30   Create   : dvomerge (in1) and (in2) to (out) -- create a new db from two input dbs
     31   Update   : dvomerge (in) into (out)          -- merge a new db into an existing db
     32   Continue : dvomerge (in) into (out) continue -- merge a new db into an existing db
    2233
    2334*/

  • trunk/Ohana/src/dvomerge/src/dvomergeCreate.c

    r28855 r29938  
    1515  char *input1, *input2, *output;
    1616  IDmapType IDmap1, IDmap2;
     17  int NsecfiltInput1, NsecfiltInput2, NsecfiltOutput;
    1718
    1819  if (strcasecmp (argv[2], "and")) dvomerge_usage();
    … …  
    3940  }
    4041  input1Photcodes = GetPhotcodeTable();
     42  NsecfiltInput1 = GetPhotcodeNsecfilt();
    4143
    4244  // Read the input2 photcodes
    … …  
    4850  }
    4951  input2Photcodes = GetPhotcodeTable();
     52  NsecfiltInput2 = GetPhotcodeNsecfilt();
    5053
    5154  if (ALTERNATE_PHOTCODE_FILE) {
    … …  
    6063    }
    6164    outputPhotcodes = GetPhotcodeTable();
     65    NsecfiltOutput = GetPhotcodeNsecfilt();
    6266
    6367    secfiltMap1 = GetSecFiltMap(outputPhotcodes, input1Photcodes);
    … …  
    7478    outputPhotcodes = input1Photcodes;
    7579    codesFilename = filename1;
     80    NsecfiltOutput = NsecfiltInput1;
    7681    // secfitMap1 can remain NULL since input1 defines the set of codes
    7782  }
    … …  
    8489  }
    8590
     91  // trigger any dependencies, if any
     92  SetPhotcodeTable(NULL);
     93
    8694  // save the output photcodes in the output catdir
    87   SetPhotcodeTable(outputPhotcodes);
     95  // SetPhotcodeTable(outputPhotcodes);
    8896  sprintf (filename, "%s/Photcodes.dat", output);
    8997  if (!check_file_access (filename, TRUE, TRUE, VERBOSE)) {
    … …  
    121129    if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
    122130
    123     if (outputPhotcodes) {
    124         SetPhotcodeTable(outputPhotcodes);
    125     }
     131    // if (outputPhotcodes) {
     132    //     SetPhotcodeTable(outputPhotcodes);
     133    // }
    126134    // load / create output catalog
    127     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
    128 
    129     if (input1Photcodes) {
    130         SetPhotcodeTable(input1Photcodes);
    131     }
     135    LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", NsecfiltOutput);
     136
     137    // if (input1Photcodes) {
     138    // SetPhotcodeTable(input1Photcodes);
     139    // }
    132140    // combine only tables at equal or larger depth
    133141     
    … …  
    138146
    139147      // load input catalog (1)
    140       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     148      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", NsecfiltInput1);
    141149
    142150      // skip empty input catalogs
    … …  
    153161    SkyListFree (inlist);
    154162
    155     if (input2Photcodes) {
    156       SetPhotcodeTable(input2Photcodes);
    157     }
     163    // if (input2Photcodes) {
     164    //   SetPhotcodeTable(input2Photcodes);
     165    // }
    158166
    159167    // load in all of the tables from input2 for this region
    … …  
    163171
    164172      // load input catalog (2)
    165       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     173      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", NsecfiltInput2);
    166174
    167175      // skip empty input catalogs

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r29001 r29938  
    33int dvomergeUpdate (int argc, char **argv) {
    44
    5   int depth;
     5  int depth, CONTINUE;
    66  off_t i, j, Ns, Ne;
    77  SkyTable *outsky, *insky;
    … …  
    1313  PhotCodeData *outputPhotcodes;
    1414  int *secfiltMap = NULL;
     15  int NsecfiltInput, NsecfiltOutput;
    1516
     17  double dtime;
     18  struct timeval start, stop;
     19  gettimeofday (&start, NULL);
     20
     21  CONTINUE = FALSE;
    1622  if (strcasecmp (argv[2], "into")) dvomerge_usage();
     23  if (argc == 5) {
     24    if (strcasecmp (argv[4], "continue")) dvomerge_usage();
     25    CONTINUE = TRUE;
     26  }
    1727
    1828  input  = argv[1];
    … …  
    3141  }     
    3242  inputPhotcodes = GetPhotcodeTable();
     43  NsecfiltInput = GetPhotcodeNsecfilt();
    3344
    3445  // since we are merging the input db into the output db, the output defines the photcode
    … …  
    4152
    4253    outputPhotcodes = GetPhotcodeTable();
     54    NsecfiltOutput = GetPhotcodeNsecfilt();
    4355
    4456    secfiltMap = GetSecFiltMap(outputPhotcodes, inputPhotcodes);
    … …  
    5062    fprintf(stderr, "%s not found using photcodes from %s/Photcodes.dat\n", filename, input);
    5163    outputPhotcodes = inputPhotcodes;
     64    NsecfiltOutput = NsecfiltInput;
     65
    5266    if (!check_dir_access (output, VERBOSE)) {
    5367      fprintf (stderr, "error creating output database directory %s\n", output);
    … …  
    6175  }
    6276
    63   dvomergeImagesUpdate (&IDmap, input, output);
     77  if (CONTINUE) {
     78    // need to determine the mapping from the input to the output images
     79    dvomergeImagesGetMap (&IDmap, input, output);
     80  } else {
     81    dvomergeImagesUpdate (&IDmap, input, output);
     82  }
    6483
    6584  // load the sky table for the existing database
    … …  
    85104  depth = inlist[0].regions[Ns][0].depth;
    86105 
     106  SetPhotcodeTable(NULL);
     107
    87108  // loop over the populated input regions
    88109  for (i = 0; i < inlist[0].Nregions; i++) {
    … …  
    90111    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    91112
    92     SetPhotcodeTable(inputPhotcodes);
    93113    // load / create output catalog (if catalog does not exist, it will be created)
    94     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r");
     114    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    95115    // skip empty input catalogs
    96116    if (!incatalog.Naves_disk) {
    … …  
    111131
    112132      // load input catalog
    113       SetPhotcodeTable(outputPhotcodes);
    114       LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w");
     133      LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    115134
    116135      dvo_update_image_IDs (&IDmap, &incatalog);
    … …  
    121140      // if we receive a signal which would cause us to exit, wait until the full catalog is written
    122141      SetProtect (TRUE);
    123       dvo_catalog_save (&outcatalog, VERBOSE);
     142      if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
     143        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
     144        exit (1);
     145      }
    124146      SetProtect (FALSE);
    125147
    … …  
    143165  }
    144166
     167  gettimeofday (&stop, NULL);
     168  dtime = DTIME (stop, start);
     169  fprintf (stderr, "SUCCESS: elapsed time %9.4f sec\n", dtime);
     170
    145171  exit (0);
    146172}
    147 
    148 /*** update the image table ***/
    149 int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) {
    150 
    151   FITS_DB inDB;
    152   FITS_DB outDB;
    153   int    status;
    154 
    155   /*** load input1/Images.dat ***/
    156   sprintf (ImageCat, "%s/Images.dat", input);
    157   inDB.mode   = dvo_catalog_catmode (CATMODE);
    158   inDB.format = dvo_catalog_catformat (CATFORMAT);
    159   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    160   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    161 
    162   // load the image table
    163   if (inDB.dbstate == LCK_EMPTY) {
    164     // Shutdown ("can't find input image catalog %s", inDB.filename);
    165     IDmap->Nmap = 0;
    166     return TRUE;
    167   }
    168   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    169     Shutdown ("can't read input image catalog %s", inDB.filename);
    170   }
    171 
    172   /*** load output/Images.dat ***/
    173   sprintf (ImageCat, "%s/Images.dat", output);
    174   outDB.mode   = dvo_catalog_catmode (CATMODE);
    175   outDB.format = dvo_catalog_catformat (CATFORMAT);
    176   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    177   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    178 
    179   /* load the image table */
    180   if (outDB.dbstate == LCK_EMPTY) {
    181     dvo_image_create (&outDB, GetZeroPoint());
    182   } else {
    183     if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    184       Shutdown ("can't read output image catalog %s", outDB.filename);
    185     }
    186   }
    187 
    188   // convert database table to internal structure & add to output image db
    189   dvo_image_merge_dbs(IDmap, &outDB, &inDB);
    190   dvo_image_unlock (&inDB); // unlock input
    191 
    192   SetProtect (TRUE);
    193   dvo_image_save (&outDB, VERBOSE);
    194   SetProtect (FALSE);
    195   dvo_image_unlock (&outDB); // unlock output
    196 
    197   return TRUE;
    198 }

  • trunk/Ohana/src/dvomerge/src/dvoverify.c

    r29001 r29938  
    88*/
    99
     10int VerifyTableFile (char *filename);
     11
     12# define DEBUG 0
     13
    1014int main (int argc, char **argv) {
    1115
    12   char filename[256], *input, *output;
    13 
    14   int depth;
    15   off_t i, j, Ns, Ne;
    16   SkyTable *outsky, *insky;
     16  char filename[1024];
     17
     18  int N, Nbad;
     19  off_t i;
     20  SkyTable *insky;
    1721  SkyList *inlist;
    18   Catalog incatalog, outcatalog;
    19 
    20   SetSignals ();
    21   dvoconvert_help (argc, argv);
    22   ConfigInit (&argc, argv);
    23   dvoconvert_args (&argc, argv);
    24 
    25   if (strcasecmp (argv[2], "to")) dvoconvert_usage();
    26   input = argv[1];
    27   output = argv[3];
    28 
    29   // load input images, save to output images
    30   dvoConvert_copy_images (input, output);
    31 
    32   // copy photcode table
    33   {
    34     // the first input defines the photcode table & db layout
    35     sprintf (filename, "%s/Photcodes.dat", input);
    36     if (!LoadPhotcodes (filename, NULL, FALSE)) {
    37       fprintf (stderr, "error loading photcode table %s\n", filename);
    38       exit (1);
    39     }
    40     // save the photcodes in the output catdir
    41     sprintf (filename, "%s/Photcodes.dat", output);
    42     if (!check_file_access (filename, TRUE, TRUE, VERBOSE)) {
    43       fprintf (stderr, "error creating output catdir %s\n", output);
    44       exit (1);
    45     }
    46     if (!SavePhotcodesFITS (filename)) {
    47       fprintf (stderr, "error saving photcode table %s\n", filename);
    48       exit (1);
    49     }
    50   }
    51 
    52   // copy skytable
    53   {
    54     // load the sky table for the existing database
    55     insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
    56     SkyTableSetFilenames (insky, input, "cpt");
    57 
    58     // generate an output table populated at the desired depth
    59     // XXX force this to match the input?
    60     outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
    61     SkyTableSetFilenames (outsky, output, "cpt");
    62 
    63     SkyTablePopulatedRange (&Ns, &Ne, insky, 0);
    64     depth = insky[0].regions[Ns].depth;
    65     // XXX this seems to imply that insky is a uniform depth...
    66   }
    67 
    68   // loop over all input catalogs, save to output catalogs
    69   // loop over the populatable output regions
    70   for (i = 0; i < outsky[0].Nregions; i++) {
    71     if (!outsky[0].regions[i].table) continue;
    72     if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
    73 
    74     // load / create output catalog
    75     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
    76 
    77     // combine only tables at equal or larger depth
    78      
    79     // load in all of the tables from input for this region
    80     inlist = SkyListByBounds (insky, depth, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
    81     for (j = 0; j < inlist[0].Nregions; j++) {
    82       if (VERBOSE) fprintf (stderr, "input : %s\n", inlist[0].regions[j][0].name);
    83 
    84       // load input catalog
    85       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
    86 
    87       // skip empty input catalogs
    88       if (!incatalog.Naves_disk) {
    89         dvo_catalog_unlock (&incatalog);
    90         dvo_catalog_free (&incatalog);
    91         continue;
    92       }
    93       merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog);
    94       dvo_catalog_unlock (&incatalog);
    95       dvo_catalog_free (&incatalog);
    96     }
    97     SkyListFree (inlist);
    98 
    99     outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    100     dvo_catalog_save (&outcatalog, VERBOSE);
    101     dvo_catalog_unlock (&outcatalog);
    102     dvo_catalog_free (&outcatalog);
    103   }
    104 
    105   // save the output sky table copy
    106   sprintf (filename, "%s/SkyTable.fits", output);
    107   check_file_access (filename, TRUE, TRUE, VERBOSE);
    108   if (!SkyTableSave (outsky, filename)) {
    109     fprintf (stderr, "ERROR: failed to save sky table for %s\n", output);
     22  SkyRegion UserPatch;
     23  // Catalog catalog;
     24
     25  // restrict to a portion of the sky
     26  UserPatch.Rmin = 0;
     27  UserPatch.Rmax = 360;
     28  UserPatch.Dmin = -90;
     29  UserPatch.Dmax = +90;
     30  if ((N = get_argument (argc, argv, "-region"))) {
     31    remove_argument (N, &argc, argv);
     32    UserPatch.Rmin = atof (argv[N]);
     33    remove_argument (N, &argc, argv);
     34    UserPatch.Rmax = atof (argv[N]);
     35    remove_argument (N, &argc, argv);
     36    UserPatch.Dmin = atof (argv[N]);
     37    remove_argument (N, &argc, argv);
     38    UserPatch.Dmax = atof (argv[N]);
     39    remove_argument (N, &argc, argv);
     40  }
     41
     42  if (argc != 2) {
     43    fprintf (stderr, "USAGE: dvoverify (catdir) [-region Rmin Rmax Dmin Dmax]\n");
     44    fprintf (stderr, "  catdir : database of interest\n");
     45    exit (2);
     46  }
     47
     48  char *catdir = argv[1];
     49
     50  Nbad = 0;
     51
     52  // XXX make this step optional
     53  if (1) {
     54    // check the photcode table
     55    sprintf (filename, "%s/Photcodes.dat", catdir);
     56    if (!VerifyTableFile (filename)) {
     57      Nbad ++;
     58    }
     59
     60    // check the skytable
     61    sprintf (filename, "%s/SkyTable.fits", catdir);
     62    if (!VerifyTableFile (filename)) {
     63      Nbad ++;
     64    }
     65
     66    // check the image table
     67    sprintf (filename, "%s/Images.dat", catdir);
     68    if (!VerifyTableFile (filename)) {
     69      Nbad ++;
     70    }
     71  }
     72
     73  // load the sky table for the existing database
     74  insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     75  myAssert(insky, "can't read SkyTable");
     76  SkyTableSetFilenames (insky, catdir, "cpt");
     77  inlist = SkyListByPatch (insky, -1, &UserPatch);
     78 
     79  // loop over all catalogs, save to output catalogs
     80  for (i = 0; i < inlist[0].Nregions; i++) {
     81    if (!inlist[0].regions[i][0].table) continue;
     82
     83    if (i % 1000 == 0) fprintf (stderr, ".");
     84
     85    sprintf (filename, "%s/%s.cpt", catdir, inlist[0].regions[i][0].name);
     86    if (!VerifyTableFile (filename)) {
     87      Nbad ++;
     88    }
     89
     90    sprintf (filename, "%s/%s.cps", catdir, inlist[0].regions[i][0].name);
     91    if (!VerifyTableFile (filename)) {
     92      Nbad ++;
     93    }
     94
     95    sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
     96    if (!VerifyTableFile (filename)) {
     97      Nbad ++;
     98    }
     99  }
     100
     101  if (Nbad > 0) {
     102    fprintf (stderr, "ERROR: %d files are bad\n", Nbad);
    110103    exit (1);
    111104  }
    112105
     106  fprintf (stderr, "SUCCESS: no files are bad\n");
    113107  exit (0);
    114108}
    115109
    116 int dvoConvert_copy_images (char *input, char *output) {
    117 
    118   FITS_DB inDB;
    119   FITS_DB outDB;
    120   int    status;
    121 
    122   Image *images;
    123   off_t Nimages;
    124   off_t ID;
    125 
    126   /*** load output/Images.dat ***/
    127   sprintf (ImageCat, "%s/Images.dat", output);
    128   outDB.mode   = dvo_catalog_catmode (CATMODE);
    129   outDB.format = dvo_catalog_catformat (CATFORMAT);
    130   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    131   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    132 
    133   // output image table should not already exist
    134   if (outDB.dbstate != LCK_EMPTY) {
    135     Shutdown ("ERROR: image table %s already exists", outDB.filename);
    136   }
    137   dvo_image_create (&outDB, GetZeroPoint());
    138 
    139   /*** load input/Images.dat ***/
    140   sprintf (ImageCat, "%s/Images.dat", input);
    141   // inDB.mode   = dvo_catalog_catmode (CATMODE);
    142   // inDB.format = dvo_catalog_catformat (CATFORMAT);
    143   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    144   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    145 
    146   // load the image table
    147   if (inDB.dbstate == LCK_EMPTY) {
    148     Shutdown ("can't find input image catalog %s", inDB.filename);
    149   }
    150   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    151     Shutdown ("can't read input image catalog %s", inDB.filename);
    152   }
    153 
    154   // convert the raw image table to Image type (byteswap if needed)
    155   images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped);
    156   if (!images) {
    157     fprintf (stderr, "ERROR: failed to read images\n");
    158     exit (2);
    159   }
    160 
    161   // update additional metadata
    162   gfits_scan (&inDB.header, "IMAGEID", OFF_T_FMT, 1,  &ID);
    163   gfits_modify (&outDB.header, "NIMAGES", OFF_T_FMT, 1,  Nimages);
    164   gfits_modify (&outDB.header, "IMAGEID", OFF_T_FMT, 1,  ID);
    165 
    166   // copy input rows to output table
    167   gfits_add_rows (&outDB.ftable, (char *) images, Nimages, sizeof(Image));
    168 
    169   // write out the image table to disk
    170   SetProtect (TRUE);
    171   dvo_image_save (&outDB, VERBOSE);
    172   SetProtect (FALSE);
    173   dvo_image_unlock (&outDB); // unlock output
    174   dvo_image_unlock (&inDB); // unlock input1
    175 
     110// is this file a consistent FITS file?
     111int VerifyTableFile (char *filename) {
     112
     113  int status;
     114  off_t Nbytes;
     115  Header header;
     116
     117  struct stat fileStats;
     118  FILE *file;
     119
     120  // does the file exist?
     121  status = stat (filename, &fileStats);
     122  if (status) {
     123    // some error accessing the file.  there is only one acceptable error: file not found
     124    switch (errno) {
     125      case ENOENT:
     126        if (DEBUG) fprintf (stderr, "file does not exist, skipping %s\n", filename);
     127        return TRUE;
     128      case ENOMEM:
     129        fprintf (stderr, "Out of memory: %s\n", filename);
     130        return TRUE;
     131      case EACCES:
     132        fprintf (stderr, "Permission error on %s\n", filename);
     133        return FALSE;
     134      case EFAULT:
     135        fprintf (stderr, "Bad address: %s\n", filename);
     136        return FALSE;
     137      case ELOOP:
     138        fprintf (stderr, "Too many symbolic links encountered while traversing the path: %s\n", filename);
     139        return FALSE;
     140      case ENAMETOOLONG:
     141        fprintf (stderr, "File name too long: %s\n", filename);
     142        return FALSE;
     143      case ENOTDIR:
     144        fprintf (stderr, "A component of the path is not a directory: %s\n", filename);
     145        return FALSE;
     146      case EOVERFLOW:
     147        fprintf (stderr, "file too large for program version: %s\n", filename);
     148        return FALSE;
     149      default:
     150        fprintf (stderr, "unknown error: %s\n", filename);
     151        return FALSE;
     152    }
     153  }
     154
     155  // does it have any data?
     156  if (fileStats.st_size == 0) {
     157    fprintf (stderr, "file is empty: %s\n", filename);
     158    return FALSE;
     159  }
     160
     161  // can we open it?
     162  file = fopen(filename, "r");
     163  if (!file) {
     164    fprintf (stderr, "unable to open valid file: %s\n", filename);
     165    return FALSE;
     166  }
     167
     168  // scan all extentions
     169  Nbytes = 0;
     170  if (DEBUG) fprintf (stderr, "sizes: ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nbytes, fileStats.st_size);
     171  while (Nbytes < fileStats.st_size) {
     172
     173    // Check on the PHU
     174    if (!gfits_fread_header (file, &header)) {
     175      fprintf (stderr, "unable to read PHU header for %s\n", filename);
     176      fclose(file);
     177      return (FALSE);
     178    }
     179
     180    // move to TBL header
     181    Nbytes += header.datasize + gfits_data_size (&header);
     182    if (DEBUG) fprintf (stderr, "sizes: ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nbytes, fileStats.st_size);
     183    if (Nbytes > fileStats.st_size) {
     184      fprintf (stderr, "file is short ("OFF_T_FMT" vs "OFF_T_FMT"): %s\n", Nbytes, fileStats.st_size, filename);
     185      gfits_free_header(&header);
     186      fclose (file);
     187      return FALSE;
     188    }
     189    gfits_free_header(&header);
     190
     191    status = fseeko (file, Nbytes, SEEK_SET);
     192    if (status) {
     193      switch (errno) {
     194        case EBADF:
     195          fprintf (stderr, "something wrong with file handle: %s\n", filename);
     196          fclose (file);
     197          return FALSE;
     198        case EINVAL:
     199          fprintf (stderr, "invalid offset: %s\n", filename);
     200          fclose (file);
     201          return FALSE;
     202        default:
     203          fprintf (stderr, "other error in fseeko: %s\n", filename);
     204          fclose (file);
     205          return FALSE;
     206      }
     207    }
     208  }
     209  if (DEBUG) fprintf (stderr, "file is good: %s\n", filename);
     210  fclose (file);
    176211  return TRUE;
    177212}
     213
     214  // gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
     215  // gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows);
     216   
     217 

  • trunk/Ohana/src/dvomerge/src/help.c

    r29001 r29938  
    44  fprintf (stderr, "USAGE: dvomerge (input1) and (input2) to (output)\n");
    55  fprintf (stderr, "   OR: dvomerge (input) into (output)\n");
     6  fprintf (stderr, "   OR: dvomerge (input) into (output) continue\n");
     7  fprintf (stderr, "   OR: dvomerge (input) into (output) from (list)\n");
    68  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
    79  exit (2);
    … …  
    1719void dvosecfilt_usage(void) {
    1820
    19   fprintf (stderr, "USAGE: dvosecfilt (input) -photcodes (photcodes.txt)\n");
     21  fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n");
    2022
    2123  exit (2);
    … …  
    3335  fprintf (stderr, "USAGE\n");
    3436  fprintf (stderr, "  dvomerge (input1) and (input2) to (output)\n");
    35   fprintf (stderr, "  dvomerge (input) into (output)\n\n");
     37  fprintf (stderr, "  dvomerge (input) into (output)\n");
     38  fprintf (stderr, "  dvomerge (input) into (output) continue\n\n");
     39  fprintf (stderr, "  dvomerge (input) into (output) from (list)\n\n");
    3640  fprintf (stderr, "  merge DVO databases\n");
    3741  fprintf (stderr, "  optional flags:\n");
    … …  
    3943  fprintf (stderr, "  -help                       : this list\n");
    4044  fprintf (stderr, "  -h                          : this list\n\n");
     45  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
    4146  exit (2);
    4247}
    … …  
    7580
    7681  fprintf (stderr, "USAGE\n");
    77   fprintf (stderr, "  dvosecfilt (input) (Nsecfilt)\n\n");
     82  fprintf (stderr, "  dvosecfilt (catdir) (Nsecfilt)\n\n");
    7883
    79   fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables only)\n");
     84  fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables (cps), NSECFILT in cpt files)\n");
     85  fprintf (stderr, "  NOTE: the user must change the photcode table to reflect the change (use photcode-table -export / -import)\n");
    8086 
    8187  fprintf (stderr, "  optional flags:\n");
    … …  
    8692}
    8793
     94void dvorepair_help (int argc, char **argv) {
     95
     96  /* check for help request */
     97  if (!argv) goto show_help;
     98  if (get_argument (argc, argv, "-help")) goto show_help;
     99  if (get_argument (argc, argv, "-h"))    goto show_help;
     100  if (argc < 2) goto show_help;
     101  return;
     102
     103show_help:
     104
     105  fprintf (stderr, "USAGE\n");
     106  fprintf (stderr, "  dvorepair -fix-cpt (images) (rootlist) - regenerate cpt & cps files from the cpm files\n");
     107  fprintf (stderr, "  dvorepair -images-vs-measures (catdir) (Ntol) - find images with too many missing detections\n");
     108  fprintf (stderr, "  dvorepair -delete-image-list (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n");
     109  fprintf (stderr, "  dvorepair -fix-images (catdir) (deleteList) - delete a set of images based on image IDs (output from -images-vs-measures)\n");
     110  fprintf (stderr, "\n");
     111
     112  fprintf (stderr, "  optional flags:\n");
     113  fprintf (stderr, "  -help                       : this list\n");
     114  fprintf (stderr, "  -h                          : this list\n\n");
     115  exit (2);
     116}
     117

  • trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c

    r29181 r29938  
    2828  unsigned int objID, catID;
    2929  Coords tcoords;
    30 
     30 
    3131  // struct timeval start, stop;
    3232  // gettimeofday (&start, (void *) NULL);
    … …  
    240240    } else {
    241241      for (k = 0; k < NsecfiltIn; k++) {
     242        if (secfiltMap[k] < 0) continue;  // skip secfilt entries from input that are not in the output
     243
    242244        // index for this entry in output's secfilt list
    243245        int outputIndex = n * NsecfiltOut + secfiltMap[k];
    … …  
    254256    i++;
    255257  }
    256   // MARKTIME("find matched stars: %f sec for "OFF_T_FMT","OFF_T_FMT" stars\n", dtime, Nstars, Nave);
     258  // MARKTIME("find matched stars: %f sec for "OFF_T_FMT","OFF_T_FMT" stars ("OFF_T_FMT" meas)\n", dtime, Nstars, Nave, Nmeas);
    257259
    258260  /** incorporate unmatched image stars, if this star is in field of this catalog **/
Note: See TracChangeset for help on using the changeset viewer.