IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 40561


Ignore:
Timestamp:
Nov 29, 2018, 1:29:46 PM (8 years ago)
Author:
eugene
Message:

update dvosplit to handle all catalog tables

Location:
trunk/Ohana/src/dvosplit
Files:
4 edited
3 copied

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvosplit/include/dvosplit.h

    r37807 r40561  
    4040
    4141Catalog   *open_output_catalogs   PROTO((SkyList *outlist, int catformat, int catmode));
    42 AveLinks  *split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
     42int        split_averages         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs));
    4343int        split_measures         PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks));
    4444void GetConfig (char *config, char *field, char *format, int N, void *ptr);

  • trunk/Ohana/src/dvosplit/src/dvosplit.c

    r38471 r40561  
    22
    33// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
    4 // dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax]
    5 // (inherites the input catalog's format and mode, unless -set-format or -set-mode are used)
     4// (inherits the input catalog's format and mode, unless -set-format or -set-mode are used)
    65int main (int argc, char **argv) {
    76
    … …  
    109  SkyList *skylist, *outlist;
    1110  Catalog incatalog, *outcatalogs;
    12   AveLinks *avelinks;
    1311  char *filename, *CATDIR;
    1412
    … …  
    1614  ConfigInit (&argc, argv);
    1715  args (argc, argv);
    18 
    19   /*
    20   fprintf (stderr,
    21   "this program has the ability to split dvo catalogs one file (cpt, cpm, etc) at a time.
    22   however, new updates to the cpm format requires the average RA,DEC coords to load the
    23   measures.  update the code to handle that first");
    24   exit (2);
    25   */
    2616
    2717  CATDIR = strcreate (argv[1]);
    … …  
    6555  for (i = 0; i < skylist[0].Nregions; i++) {
    6656    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
     57    fprintf (stderr, "reading from %s\n", skylist[0].regions[i][0].name);
    6758
    6859    // if !LocalCopy, always copy the file
    … …  
    8172    incatalog.filename = skylist[0].filename[i];
    8273    incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
    83     incatalog.catflags = DVO_LOAD_NONE;
     74    incatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT;
     75    // load all of the tables at once
    8476
    8577    // an error exit status here is a significant error
    … …  
    9789
    9890    // change sky.regions[i].depth for these regions
    99     outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
     91    // outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]);
     92    outlist = SkyListChildrenByBounds (sky, skylist[0].regions[i][0].index, OUT_DEPTH,
     93                                       skylist[0].regions[i][0].Rmin + 0.01, skylist[0].regions[i][0].Rmax - 0.01,
     94                                       skylist[0].regions[i][0].Dmin + 0.01, skylist[0].regions[i][0].Dmax - 0.01 );
    10095
    10196    // Modify the outlist filenames to match the output directory.  Note that the
    … …  
    106101    outcatalogs = open_output_catalogs (outlist, incatalog.catformat, incatalog.catmode);
    107102
    108     avelinks = split_averages (&incatalog, outlist, outcatalogs);
    109 
    110     split_measures (&incatalog, outlist, outcatalogs, avelinks);
    111 
    112     // XXX missing entries have to be reconstructed if they are desired
    113     // split_missings (&incatalog, outlist, outcatalogs, avelinks);
    114 
    115     free (avelinks[0].outref);
    116     free (avelinks[0].outcat);
     103    split_averages (&incatalog, outlist, outcatalogs);
    117104
    118105    dvo_catalog_unlock (&incatalog);
    119106
    120107    for (j = 0; j < outlist[0].Nregions; j++) {
     108      outcatalogs[j].sorted = TRUE; // split_averages generates a sorted database
     109      fprintf (stderr, "save: %s\n", outcatalogs[j].filename);
     110      dvo_catalog_save (&outcatalogs[j], VERBOSE);
    121111      dvo_catalog_unlock (&outcatalogs[j]);
    122112    }
    … …  
    146136
    147137  if (!LocalCopy) {
    148     // copy the images table
     138    // copy the images table (there are others we should copy as well)
    149139    char line[2048];
    150140    snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR);

  • trunk/Ohana/src/dvosplit/src/dvosplit.subset.c

    r40560 r40561  
    22
    33// dvosplit (catdir) (outlevel) [-outdir (outcat)} [-region Rmin Rmax Dmin Dmax]
    4 // dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax]
    5 // (inherites the input catalog's format and mode, unless -set-format or -set-mode are used)
     4// (inherits the input catalog's format and mode, unless -set-format or -set-mode are used)
    65int main (int argc, char **argv) {
    76
    … …  
    1615  ConfigInit (&argc, argv);
    1716  args (argc, argv);
    18 
    19   /*
    20   fprintf (stderr,
    21   "this program has the ability to split dvo catalogs one file (cpt, cpm, etc) at a time.
    22   however, new updates to the cpm format requires the average RA,DEC coords to load the
    23   measures.  update the code to handle that first");
    24   exit (2);
    25   */
    2617
    2718  CATDIR = strcreate (argv[1]);

  • trunk/Ohana/src/dvosplit/src/open_output_catalogs.c

    r38471 r40561  
    1515    outcatalogs[i].filename  = outlist[0].filename[i];
    1616    outcatalogs[i].Nsecfilt  = GetPhotcodeNsecfilt ();
    17     outcatalogs[i].catflags  = DVO_LOAD_NONE;
     17    outcatalogs[i].catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE | DVO_LOAD_LENSING | DVO_LOAD_LENSOBJ | DVO_LOAD_STARPAR | DVO_LOAD_GALPHOT; // for a subset, use: DVO_LOAD_NONE;
    1818    outcatalogs[i].catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : catformat;  // set the default catformat from config data
    1919    outcatalogs[i].catmode   = CATMODE   ? dvo_catalog_catmode (CATMODE)     : catmode;    // set the default catmode from config data

  • trunk/Ohana/src/dvosplit/src/split_averages.c

    r38986 r40561  
    33# define DNOUT 1000
    44
    5 AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
     5/* used in find_matches, find_matches_refstars */
     6# define IN_REGION(REG,R,D) (                        \
     7((D) >= REG[0].Dmin) && ((D) < REG[0].Dmax) && \
     8((R) >= REG[0].Rmin) && ((R) < REG[0].Rmax))
    69
    7   double inR, inD;
    8   int n, block, ave, cat, averef, Nblocks, Ncat, Nout, Nsecfilt;
    9   int *outref, *outcat, *outmem;
    10   AveLinks *avelinks;
     10// incatalog has already been loaded; we now need to split out the entries to the subcatalogs
     11// outcatalog[] have already been opened (and are starting at empty)
    1112
    12   ALLOCATE (outref, int, incatalog[0].Naverage_disk);
    13   ALLOCATE (outcat, int, incatalog[0].Naverage_disk);
    14   ALLOCATE (outmem, int, outlist[0].Nregions);
     13// this version requires the entire catalog in memory at once
     14int split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
    1515
    16   Nsecfilt = GetPhotcodeNsecfilt ();
     16  off_t Nm; // used to track number of values for a given object in a table
    1717
    18   // allocate enough space for these output buffers: use Nsecfilt + 1 incase the file
    19   // contains primary photcodes, which will increase Nsecfilt by one.
    20   for (cat = 0; cat < outlist[0].Nregions; cat++) {
    21     outmem[cat] = DNOUT;
    22     REALLOCATE (outcatalogs[cat].average, Average, outmem[cat]);
    23     REALLOCATE (outcatalogs[cat].secfilt, SecFilt, outmem[cat]*(Nsecfilt + 1));
     18  // we need a list of currently-allocated elements for each data type:
     19  ALLOCATE_PTR (NAVERAGE, int, outlist[0].Nregions);
     20  ALLOCATE_PTR (NMEASURE, int, outlist[0].Nregions);
     21  ALLOCATE_PTR (NLENSING, int, outlist[0].Nregions);
     22  ALLOCATE_PTR (NSTARPAR, int, outlist[0].Nregions);
     23  ALLOCATE_PTR (NGALPHOT, int, outlist[0].Nregions);
     24
     25  int Nsecfilt = GetPhotcodeNsecfilt ();
     26
     27  for (int cat = 0; cat < outlist[0].Nregions; cat++) {
     28    NAVERAGE[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].average, Average, NAVERAGE[cat]);
     29    NMEASURE[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].measure, Measure, NMEASURE[cat]);
     30    NLENSING[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].lensing, Lensing, NLENSING[cat]);
     31    NSTARPAR[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].starpar, StarPar, NSTARPAR[cat]);
     32    NGALPHOT[cat] = DNOUT;  REALLOCATE (outcatalogs[cat].galphot, GalPhot, NGALPHOT[cat]);
     33
     34    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, NAVERAGE[cat]*Nsecfilt);
    2435  }
    2536
    26   // split out the average & secfilt entries:
    27   incatalog[0].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
     37  // distribute data to the output catalogs
     38  for (off_t ave = 0; ave < incatalog[0].Naverage; ave++) {
     39    double inR = incatalog[0].average[ave].R;
     40    double inD = incatalog[0].average[ave].D;
    2841
    29   if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
    30     Nblocks = incatalog[0].Naverage_disk / NROWS;
    31     if (incatalog[0].Naverage_disk % NROWS) Nblocks ++;
    32   } else {
    33     Nblocks = 1;
     42    // which of the outcatalogs contains this coordinate?
     43    int Ncat = -1;
     44    for (int cat = 0; cat < outlist[0].Nregions; cat++) {
     45      if (!IN_REGION(outlist[0].regions[cat], inR, inD)) continue;
     46      Ncat = cat;
     47      break;
     48    }
     49
     50    // NO outcatalogs contains this coordinate?
     51    if (Ncat == -1) {
     52      fprintf (stderr, "WARNING: missed "OFF_T_FMT" (%f, %f)\n", ave, inR, inD);
     53      continue;
     54    }
     55   
     56    // these values are the current (next) entry for the table
     57    off_t averageOut = outcatalogs[Ncat].Naverage;
     58    off_t measureOut = outcatalogs[Ncat].Nmeasure;
     59    off_t lensingOut = outcatalogs[Ncat].Nlensing;
     60    off_t starparOut = outcatalogs[Ncat].Nstarpar;
     61    off_t galphotOut = outcatalogs[Ncat].Ngalphot;
     62
     63    // assign the value to the next element of the output catalog
     64    outcatalogs[Ncat].average[averageOut] = incatalog[0].average[ave];
     65
     66    // fprintf (stderr, "catalog %s: aveOut: %d m
     67
     68    // these values track the start of the values for the table for this object
     69    outcatalogs[Ncat].average[averageOut].measureOffset = measureOut;
     70    outcatalogs[Ncat].average[averageOut].lensingOffset = lensingOut;
     71    outcatalogs[Ncat].average[averageOut].starparOffset = starparOut;
     72    outcatalogs[Ncat].average[averageOut].galphotOffset = galphotOut;
     73   
     74    // update secfilt at the same time
     75    for (int n = 0; n < Nsecfilt; n++) {
     76      outcatalogs[Ncat].secfilt[averageOut*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n];
     77    }
     78    outcatalogs[Ncat].Naverage ++;
     79   
     80    if (outcatalogs[Ncat].Naverage >= NAVERAGE[Ncat]) {
     81      NAVERAGE[Ncat] += DNOUT;
     82      REALLOCATE (outcatalogs[Ncat].average, Average, NAVERAGE[Ncat]);
     83      REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, NAVERAGE[Ncat]*Nsecfilt);
     84    }
     85
     86    // assign the Measure values for this object to the output catalog
     87    Nm = 0;
     88    for (int j = 0; j < incatalog[0].average[ave].Nmeasure; j++) {
     89      off_t offset = incatalog[0].average[ave].measureOffset + j;
     90
     91      outcatalogs[Ncat].measure[measureOut] = incatalog[0].measure[offset];
     92      outcatalogs[Ncat].measure[measureOut].averef = averageOut;
     93
     94      measureOut ++;
     95      Nm ++;
     96      if (measureOut >= NMEASURE[Ncat]) {
     97        NMEASURE[Ncat] += DNOUT;
     98        REALLOCATE (outcatalogs[Ncat].measure, Measure, NMEASURE[Ncat]);
     99      }
     100    }
     101    outcatalogs[Ncat].average[averageOut].Nmeasure = Nm;
     102
     103    Nm = 0;
     104    for (int j = 0; j < incatalog[0].average[ave].Nlensing; j++) {
     105      off_t offset = incatalog[0].average[ave].lensingOffset + j;
     106
     107      outcatalogs[Ncat].lensing[lensingOut] = incatalog[0].lensing[offset];
     108      outcatalogs[Ncat].lensing[lensingOut].averef = averageOut;
     109
     110      lensingOut ++;
     111      Nm ++;
     112      if (lensingOut >= NLENSING[Ncat]) {
     113        NLENSING[Ncat] += DNOUT;
     114        REALLOCATE (outcatalogs[Ncat].lensing, Lensing, NLENSING[Ncat]);
     115      }
     116    }
     117    outcatalogs[Ncat].average[averageOut].Nlensing = Nm;
     118
     119    Nm = 0;
     120    for (int j = 0; j < incatalog[0].average[ave].Nstarpar; j++) {
     121      off_t offset = incatalog[0].average[ave].starparOffset + j;
     122
     123      outcatalogs[Ncat].starpar[starparOut] = incatalog[0].starpar[offset];
     124      outcatalogs[Ncat].starpar[starparOut].averef = averageOut;
     125
     126      starparOut ++;
     127      Nm ++;
     128      if (starparOut == NSTARPAR[Ncat]) {
     129        NSTARPAR[Ncat] += DNOUT;
     130        REALLOCATE (outcatalogs[Ncat].starpar, StarPar, NSTARPAR[Ncat]);
     131      }
     132    }
     133    outcatalogs[Ncat].average[averageOut].Nstarpar = Nm;
     134
     135    Nm = 0;
     136    for (int j = 0; j < incatalog[0].average[ave].Ngalphot; j++) {
     137      off_t offset = incatalog[0].average[ave].galphotOffset + j;
     138
     139      outcatalogs[Ncat].galphot[galphotOut] = incatalog[0].galphot[offset];
     140      outcatalogs[Ncat].galphot[galphotOut].averef = averageOut;
     141
     142      galphotOut ++;
     143      Nm ++;
     144      if (galphotOut == NGALPHOT[Ncat]) {
     145        NGALPHOT[Ncat] += DNOUT;
     146        REALLOCATE (outcatalogs[Ncat].galphot, GalPhot, NGALPHOT[Ncat]);
     147      }
     148    }
     149    outcatalogs[Ncat].average[averageOut].Ngalphot = Nm;
     150
     151    outcatalogs[Ncat].Nmeasure = measureOut;
     152    outcatalogs[Ncat].Nlensing = lensingOut;
     153    outcatalogs[Ncat].Nstarpar = starparOut;
     154    outcatalogs[Ncat].Ngalphot = galphotOut;
     155
    34156  }
    35157
    36   for (block = 0; block < Nblocks; block++) {
     158  free (NAVERAGE);
     159  free (NMEASURE);
     160  free (NLENSING);
     161  free (NSTARPAR);
     162  free (NGALPHOT);
    37163
    38     if ((incatalog[0].catmode == DVO_MODE_SPLIT) && !FULL_TABLE) {
    39       // read up to NROWS at a time
    40       dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
    41       fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
    42       assert (block*NROWS == incatalog[0].Naverage_off);
    43     } else {
    44       dvo_catalog_load (incatalog, VERBOSE);
    45       fprintf (stderr, "splitting %s (averages)\n", incatalog[0].filename);
    46     }
    47 
    48     // distribute data to the output catalogs
    49     for (ave = 0; ave < incatalog[0].Naverage; ave++) {
    50       averef = ave + incatalog[0].Naverage_off;
    51        
    52       inR = incatalog[0].average[ave].R;
    53       inD = incatalog[0].average[ave].D;
    54 
    55       if (incatalog[0].average[ave].Nmeasure == 0) {
    56         fprintf (stderr, "WARNING: object with no measurements, skipping %d (%f, %f)\n", averef, inR, inD);
    57         continue;
    58       }
    59       // which of the outcatalogs contains this coordinate?
    60 
    61       Ncat = -1;
    62       for (cat = 0; cat < outlist[0].Nregions; cat++) {
    63         if (inR < outlist[0].regions[cat][0].Rmin) continue;
    64         if (inR > outlist[0].regions[cat][0].Rmax) continue;
    65         if (inD < outlist[0].regions[cat][0].Dmin) continue;
    66         if (inD > outlist[0].regions[cat][0].Dmax) continue;
    67         Ncat = cat;
    68         break;
    69       }
    70 
    71       if (Ncat == -1) {
    72         fprintf (stderr, "WARNING: missed %d (%f, %f)\n", averef, inR, inD);
    73         continue;
    74       }
    75 
    76       Nout = outcatalogs[Ncat].Naverage;
    77       outref[averef] = Nout + outcatalogs[Ncat].Naverage_off;
    78       outcat[averef] = Ncat;
    79 
    80       // assign the value to the next element of the output catalog
    81       outcatalogs[Ncat].average[Nout] = incatalog[0].average[ave];
    82       outcatalogs[Ncat].Naverage ++;
    83 
    84       // update secfilt at the same time
    85       for (n = 0; n < Nsecfilt; n++) {
    86         outcatalogs[Ncat].secfilt[Nout*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n];
    87         outcatalogs[Ncat].Nsecfilt_mem++;
    88       }
    89 
    90       if (outcatalogs[Ncat].Naverage >= outmem[Ncat]) {
    91         outmem[Ncat] += DNOUT;
    92         REALLOCATE (outcatalogs[Ncat].average, Average, outmem[Ncat]);
    93         REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, outmem[Ncat]*(Nsecfilt + 1));
    94       }
    95     }
    96     dvo_catalog_free_data (incatalog);
    97 
    98     // double check the values of Naverage, Nsecfilt_mem?
    99 
    100     // XXX for output.catformat == MEF, we probably need to skip this stuff and the free below
    101 
    102     // write out the new values
    103     if (!FULL_TABLE) {
    104       for (cat = 0; cat < outlist[0].Nregions; cat++) {
    105         outcatalogs[cat].catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
    106 
    107         SetProtect (TRUE);
    108         dvo_catalog_save (&outcatalogs[cat], VERBOSE);
    109         SetProtect (FALSE);
    110         // fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecfilt_mem, outcatalogs[cat].Nsecfilt_disk, outcatalogs[cat].Nsecfilt_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
    111 
    112         // advance the pointers and free the current data
    113         // XXX these should be done within save segment:
    114         outcatalogs[cat].Naverage_disk += outcatalogs[cat].Naverage;
    115         outcatalogs[cat].Naverage_off  += outcatalogs[cat].Naverage;
    116         outcatalogs[cat].Nsecfilt_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
    117         outcatalogs[cat].Nsecfilt_off  += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage;
    118         outcatalogs[cat].Nsecfilt    = Nsecfilt;
    119 
    120         outcatalogs[cat].Naverage    = 0;
    121         outcatalogs[cat].Nsecfilt_mem   = 0;
    122       }
    123     }
    124   }
    125 
    126   if (!FULL_TABLE) {
    127     for (cat = 0; cat < outlist[0].Nregions; cat++) {
    128       dvo_catalog_free_data (&outcatalogs[cat]);
    129     }
    130   }
    131 
    132   free (outmem);
    133 
    134   ALLOCATE (avelinks, AveLinks, 1);
    135   avelinks[0].outref = outref;
    136   avelinks[0].outcat = outcat;
    137 
    138 
    139   return (avelinks);
     164  return TRUE;
    140165}

  • trunk/Ohana/src/dvosplit/src/split_averages.subset.c

    r40560 r40561  
    22# define NROWS 1000000 /* ~10MB per block for measures */
    33# define DNOUT 1000
     4
     5/* used in find_matches, find_matches_refstars */
     6# define IN_REGION(REG,R,D) (                        \
     7((D) >= REG[0].Dmin) && ((D) < REG[0].Dmax) && \
     8((R) >= REG[0].Rmin) && ((R) < REG[0].Rmax))
     9
    410
    511AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
    … …  
    5359      inD = incatalog[0].average[ave].D;
    5460
     61      // XXX do not skip : galphot dvo has Nmeasure == 0
    5562      if (incatalog[0].average[ave].Nmeasure == 0) {
    5663        fprintf (stderr, "WARNING: object with no measurements, skipping %d (%f, %f)\n", averef, inR, inD);
    57         continue;
    5864      }
     65
    5966      // which of the outcatalogs contains this coordinate?
    60 
    6167      Ncat = -1;
    6268      for (cat = 0; cat < outlist[0].Nregions; cat++) {
    63         if (inR < outlist[0].regions[cat][0].Rmin) continue;
    64         if (inR > outlist[0].regions[cat][0].Rmax) continue;
    65         if (inD < outlist[0].regions[cat][0].Dmin) continue;
    66         if (inD > outlist[0].regions[cat][0].Dmax) continue;
     69        if (!IN_REGION(outlist[0].regions[cat], inR, inD)) continue;
    6770        Ncat = cat;
    6871        break;
    6972      }
    7073
     74      // NO outcatalogs contains this coordinate?
    7175      if (Ncat == -1) {
    7276        fprintf (stderr, "WARNING: missed %d (%f, %f)\n", averef, inR, inD);
Note: See TracChangeset for help on using the changeset viewer.