Index: trunk/Ohana/src/dvosplit/Makefile =================================================================== --- trunk/Ohana/src/dvosplit/Makefile (revision 15743) +++ trunk/Ohana/src/dvosplit/Makefile (revision 15743) @@ -0,0 +1,48 @@ +default: dvosplit +help: + @echo "make options: dvosplit (default)" + +include ../../Makefile.System +HOME = $(ROOT)/src/dvosplit +BIN = $(HOME)/bin +LIB = $(HOME)/lib +SRC = $(HOME)/src +MAN = $(HOME)/doc +INC = $(HOME)/include +include ../../Makefile.Common + +# programs may add their own internal requirements here +FULL_CFLAGS = $(BASE_CFLAGS) +FULL_CPPFLAGS = $(BASE_CPPFLAGS) +FULL_LDFLAGS = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS) + +dvosplit : $(BIN)/dvosplit.$(ARCH) +all: dvosplit + +DVOSPLIT = \ +$(SRC)/dvosplit.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/split_averages.$(ARCH).o \ +$(SRC)/split_measures.$(ARCH).o \ +$(SRC)/open_output_catalogs.$(ARCH).o + +$(DVOSPLIT) : $(INC)/dvosplit.h + +$(BIN)/dvosplit.$(ARCH) : $(DVOSPLIT) + +INSTALL = dvosplit + +# dependancy rules for binary code ######################### +$(INSTALL): % : $(BIN)/%.$(ARCH) + +%.clean : + rm -f $(BIN)/$*.$(ARCH) + +%.install: + make $(DESTBIN)/$* + +install: + for i in $(INSTALL); do make $$i.install || exit; done Index: trunk/Ohana/src/dvosplit/include/dvosplit.h =================================================================== --- trunk/Ohana/src/dvosplit/include/dvosplit.h (revision 15743) +++ trunk/Ohana/src/dvosplit/include/dvosplit.h (revision 15743) @@ -0,0 +1,41 @@ +# include +# include +# include +# include +# include +# include + +/* solaris requires both of these instead of ip.h: + # include + # include +*/ + +/* linux is happy with this, not solaris */ +# include +# include +# include +# include + +typedef struct { + int *outref; + int *outcat; +} AveLinks; + +int VERBOSE; +char CATDIR[256]; +char CATMODE[16]; /* raw, mef, split, mysql */ +char CATFORMAT[16]; /* internal, elixir, loneos, panstarrs */ + +SkyRegion UserPatch; // used by MODE CAT + +int main PROTO((int argc, char **argv)); + +int ConfigInit PROTO((int *argc, char **argv)); +int SetSignals PROTO(()); +void SetProtect PROTO((int mode)); +void TrapSignal PROTO((int sig)); +int args PROTO((int argc, char **argv)); + +Catalog *open_output_catalogs PROTO((SkyList *outlist)); +AveLinks *split_averages PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs)); +int split_measures PROTO((Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks)); Index: trunk/Ohana/src/dvosplit/src/ConfigInit.c =================================================================== --- trunk/Ohana/src/dvosplit/src/ConfigInit.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/ConfigInit.c (revision 15743) @@ -0,0 +1,57 @@ +# include "dvosplit.h" + +void GetConfig (char *config, char *field, char *format, int N, void *ptr); + +int ConfigInit (int *argc, char **argv) { + + double ZERO_POINT; + char *config, *file; + char CatdirPhotcodeFile[256]; + char MasterPhotcodeFile[256]; + + /*** load configuration info ***/ + file = SelectConfigFile (argc, argv, "ptolemy"); + config = LoadConfigFile (file); + if (config == (char *) NULL) { + fprintf (stderr, "ERROR: can't find configuration file %s\n", file); + if (file != (char *) NULL) free (file); + exit (1); + } + if (VERBOSE) fprintf (stderr, "loaded config file: %s\n", file); + + GetConfig (config, "CATDIR", "%s", 0, CATDIR); + GetConfig (config, "PHOTCODE_FILE", "%s", 0, MasterPhotcodeFile); + + ScanConfig (config, "CATMODE", "%s", 0, CATMODE); + ScanConfig (config, "CATFORMAT", "%s", 0, CATFORMAT); + + /* default mode, format, if not specified */ + if (*CATMODE == 0) strcpy (CATMODE, "RAW"); + if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR"); + + GetConfig (config, "ZERO_PT", "%lf", 0, &ZERO_POINT); + SetZeroPoint (ZERO_POINT); + + /* XXX this does not yet write out the master photcode table */ + sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); + if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile)) { + fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); + exit (1); + } + + free (config); + free (file); + return (TRUE); +} + +void GetConfig (char *config, char *field, char *format, int N, void *ptr) { + + char *status; + + status = ScanConfig (config, field, format, N, ptr); + if (status == NULL) { + fprintf (stderr, "error in config, cannot find %s\n", field); + exit (1); + } + return; +} Index: trunk/Ohana/src/dvosplit/src/SetSignals.c =================================================================== --- trunk/Ohana/src/dvosplit/src/SetSignals.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/SetSignals.c (revision 15743) @@ -0,0 +1,122 @@ +# include "dvosplit.h" + +static int Protect = FALSE; +static int Trapped = FALSE; + +void TrapSignal (int sig) { + fprintf (stderr, "trapped signal %d\n", sig); + if (sig == 11) { + fprintf (stderr, "seg fault\n"); + exit (1); + } + if (Protect) { + Trapped = TRUE; + fprintf (stderr, "blocking until protected sections are clear\n"); + return; + } + Shutdown ("halted by signal (trapped)"); +} + +void SetProtect (int mode) { + Protect = mode; + if (Trapped && !Protect) Shutdown ("halted by signal (protect)"); +} + +int SetSignals () { + + int i; + + /* disable almost all signal interrupts */ + for (i = 0; i < 36; i++) { + switch (i) { + /* can't redirect these signals */ + case SIGKILL: /* kill -9: cannot be caught or ignored (POSIX.1-1990) */ + case SIGSTOP: /* SIGSTOP: cannot be caught or ignored (POSIX.1-1990) */ + /* ignore these signals */ + case SIGCHLD: /* child halted: ignore (POSIX.1-1990) */ + case SIGCONT: /* continue - maintain this action (POSIX.1-1990) */ + case SIGTSTP: /* stop signal sent from tty - why ignore? (POSIX.1-1990) */ + case SIGURG: /* socket signal, ignore this (POSIX.1-2001) */ +# ifdef SIGPWR + case SIGPWR: /* power failure - why ignore this? (Sys V) */ +# endif +# ifdef SIGWINCH + case SIGWINCH: /* window resized (4.3BSD) */ +# endif + break; + + default: + signal (i, TrapSignal); + } + } + return (TRUE); +} +/* + + Signal Value Action Comment + ------------------------------------------------------------------------- + SIGHUP 1 A Hangup detected on controlling terminal + or death of controlling process + SIGINT 2 A Interrupt from keyboard + SIGQUIT 3 A Quit from keyboard + SIGILL 4 A Illegal Instruction + SIGABRT 6 C Abort signal from abort(3) + SIGFPE 8 C Floating point exception + SIGKILL 9 AEF Kill signal + SIGSEGV 11 C Invalid memory reference + SIGPIPE 13 A Broken pipe: write to pipe with no readers + SIGALRM 14 A Timer signal from alarm(2) + SIGTERM 15 A Termination signal + SIGUSR1 30,10,16 A User-defined signal 1 + SIGUSR2 31,12,17 A User-defined signal 2 + SIGCHLD 20,17,18 B Child stopped or terminated + SIGCONT 19,18,25 Continue if stopped + SIGSTOP 17,19,23 DEF Stop process + SIGTSTP 18,20,24 D Stop typed at tty + SIGTTIN 21,21,26 D tty input for background process + SIGTTOU 22,22,27 D tty output for background process + + Next various other signals. + + Signal Value Action Comment + --------------------------------------------------------------------- + SIGTRAP 5 CG Trace/breakpoint trap + SIGIOT 6 CG IOT trap. A synonym for SIGABRT + SIGEMT 7,-,7 G + SIGBUS 10,7,10 AG Bus error + SIGSYS 12,-,12 G Bad argument to routine (SVID) + SIGSTKFLT -,16,- AG Stack fault on coprocessor + SIGURG 16,23,21 BG Urgent condition on socket (4.2 BSD) + SIGIO 23,29,22 AG I/O now possible (4.2 BSD) + SIGPOLL AG A synonym for SIGIO (System V) + SIGCLD -,-,18 G A synonym for SIGCHLD + SIGXCPU 24,24,30 AG CPU time limit exceeded (4.2 BSD) + SIGXFSZ 25,25,31 AG File size limit exceeded (4.2 BSD) + SIGVTALRM 26,26,28 AG Virtual alarm clock (4.2 BSD) + SIGPROF 27,27,29 AG Profile alarm clock + SIGPWR 29,30,19 AG Power failure (System V) + SIGINFO 29,-,- G A synonym for SIGPWR + SIGLOST -,-,- AG File lock lost + SIGWINCH 28,28,20 BG Window resize signal (4.3 BSD, Sun) + SIGUNUSED -,31,- AG Unused signal + (Here - denotes that a signal is absent; there where three values are given, the first one is usually valid for alpha and + sparc, the middle one for i386 and ppc, the last one for mips. Signal 29 is SIGINFO / SIGPWR on an alpha but SIGLOST on a + sparc.) + + The letters in the "Action" column have the following meanings: + + A Default action is to terminate the process. + + B Default action is to ignore the signal. + + C Default action is to dump core. + + D Default action is to stop the process. + + E Signal cannot be caught. + + F Signal cannot be ignored. + + G Not a POSIX.1 conformant signal. + +*/ Index: trunk/Ohana/src/dvosplit/src/Shutdown.c =================================================================== --- trunk/Ohana/src/dvosplit/src/Shutdown.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/Shutdown.c (revision 15743) @@ -0,0 +1,20 @@ +# include "dvosplit.h" + +/* clean up open / locked ImageCat before shutting down */ +int Shutdown (char *format, ...) { + va_list argp; + char *formatplus; + + ALLOCATE (formatplus, char, strlen(format) + 2); + strcpy (formatplus, format); + strcat (formatplus, "\n"); + + va_start (argp, format); + vfprintf (stderr, formatplus, argp); + free (formatplus); + va_end (argp); + + fprintf (stderr, "ERROR: addstar halted\n"); + exit (1); +} + Index: trunk/Ohana/src/dvosplit/src/args.c =================================================================== --- trunk/Ohana/src/dvosplit/src/args.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/args.c (revision 15743) @@ -0,0 +1,87 @@ +# include "dvosplit.h" +static void help (void); + +int args (int argc, char **argv) { + + int i, N, CONFIRM; + + /* check for help request */ + if (get_argument (argc, argv, "-help") || + get_argument (argc, argv, "-h")) { + help (); + } + + /*** check for command line options ***/ + + /*** provide additional data ***/ + /* restrict to a portion of the sky? (REFCAT only) */ + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + CONFIRM = TRUE; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + CONFIRM = FALSE; + } + + /* extra error messages */ + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + if (argc == 2) { + if (CONFIRM) { + fprintf (stderr, "you are splitting the entire sky in one pass\n"); + fprintf (stderr, "this could be a time consuming operation. type Ctrl-C within 5 seconds to cancel\n"); + for (i = 5; i > 0; i--) { + fprintf (stderr, "%d.. ", i); + usleep (1000000); + } + fprintf (stderr, "\n"); + } + return (TRUE); + } + + fprintf (stderr, "USAGE: dvosplit (newlevel) [-region (Rmin) (Rmax) (Dmin) (Dmax)]\n"); + exit (2); +} + +static void help () { + + fprintf (stderr, "USAGE\n"); + fprintf (stderr, " dvosplit (newlevel)\n\n"); + fprintf (stderr, " optional flags:\n"); + fprintf (stderr, " -region ra ra dec dec : migrate catalogs in specified region\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -help : this list\n"); + fprintf (stderr, " -h : this list\n\n"); + exit (2); +} + +/** addstar modes: + + addstar (image.smp) - add cmp/smp image data to db + addstar -ref (file.dat) (photcode) + addstar -cat (USNO/2MASS/GSC) -region (ra dec - ra dec) + + -replace : ref/cat - replace existing match (photcode/time) + -match : ref/cat - only add measures to existing averages + + ref types: + ASCII - RA,DEC,M,dM in a table + + addstar + +**/ + Index: trunk/Ohana/src/dvosplit/src/dvosplit.c =================================================================== --- trunk/Ohana/src/dvosplit/src/dvosplit.c (revision 15722) +++ trunk/Ohana/src/dvosplit/src/dvosplit.c (revision 15743) @@ -1,11 +1,19 @@ # include "dvosplit.h" -# define NROWS 100000 /* ~10MB per row for measures */ +// dvosplit (outlevel) [-region Rmin Rmax Dmin Dmax] int main (int argc, char **argv) { - // USAGE: dvosplit (outlevel) [-region RA RA DEC DEC] + int i, j, OUT_DEPTH; + SkyTable *sky; + SkyList *skylist, *outlist; + Catalog incatalog, *outcatalogs; + AveLinks *avelinks; + char filename[256]; - // the output catalog needs to inherit the SKY_DEPTH of the input catalog. - // if the CATDIR/SkyTable.fits is not found, assume the HST layout (old default) + SetSignals (); + ConfigInit (&argc, argv); + args (argc, argv); + + OUT_DEPTH = atoi (argv[1]); // load the sky table for the existing database @@ -13,11 +21,6 @@ SkyTableSetFilenames (sky, CATDIR, "cpt"); - // XXX I should just be copying the input sky and setting the output names on that - // load the sky table for the existing database, generate output names - outsky = SkyTableLoadOptimal (CATDIR, NULL, NULL, SKY_DEPTH_HST, VERBOSE); - SkyTableSetFilenames (outsky, OUTDIR, "cpt"); - // get the list of populated regions - skylist = SkyListByPatch (sky, -1, ®ION); + skylist = SkyListByPatch (sky, -1, &UserPatch); for (i = 0; i < skylist[0].Nregions; i++) { @@ -27,6 +30,4 @@ // if (current level == out level) skip: no action is needed if (skylist[0].regions[i][0].depth >= OUT_DEPTH) continue; - - outlist = SkyListByPatch (outsky, OUT_DEPTH, skylist[0].regions[i]); // set the parameters which guide catalog open/load/create @@ -42,5 +43,5 @@ // skip empty input catalogs - if (!incatalog.Nave_disk) { + if (!incatalog.Naves_disk) { dvo_catalog_unlock (&incatalog); dvo_catalog_free (&incatalog); @@ -48,93 +49,38 @@ } - outcatalog = SkyTableSubdivide (&incatalog, &Noutcatalog); + // change sky.regions[i].depth for these regions + outlist = SkyListByPatch (sky, OUT_DEPTH, skylist[0].regions[i]); - // split out the average entries: - incatalog.catflags = LOAD_AVES; - Nblocks = incatalog.Nave_disk / NROWS; - for (j = 0; j < Nblocks; j++) { + outcatalogs = open_output_catalogs (outlist); - // read up to NROWS at a time - dvo_catalog_load_segment (&incatalog, VERBOSE, j*NROWS, NROWS); + avelinks = split_averages (&incatalog, outlist, outcatalogs); - for (k = 0; k < incatalog.Naverage; k++) { - averef = j*NROWS + k; - - inR = incatalog.average[k].R; - inD = incatalog.average[k].D; + split_measures (&incatalog, outlist, outcatalogs, avelinks); - // which of the outcatalogs contains this coordinate? + // XXX missing entries have to be reconstructed if they are desired + // split_missings (&incatalog, outlist, outcatalogs, avelinks); - N = -1; - for (n = 0; n < Noutcatalog; n++) { - if (inR < outregions[n].Rmin) continue; - if (inR > outregions[n].Rmax) continue; - if (inD < outregions[n].Dmin) continue; - if (inD > outregions[n].Dmax) continue; - Ncat = n; - break; - } - if (Ncat == -1) continue; + dvo_catalog_unlock (&incatalog); - // XXX this probably needs to be Nave_disk so we can have partial saves - Nout = outcatalog[Ncat].Naverage; - outref[averef] = Nout; - outcat[averef] = Ncat; - - outcatalog[Ncat].average[Nout] = incatalog.average[k]; - outcatalog[Ncat].Naverage++; - } - - for (n = 0; n < Noutcatalog; n++) { - dvo_catalog_save_segment (&outcatalog[Ncat], VERBOSE, outcatalog[Ncat].Nave_disk, outcatalog[Ncat].Naverage); - } + for (j = 0; j < outlist[0].Nregions; j++) { + dvo_catalog_unlock (&outcatalogs[j]); } - // split out the measure entries: - incatalog.catflags = LOAD_MEAS; - Nblocks = incatalog.Nmeas_disk / NROWS; - for (j = 0; j < Nblocks; j++) { - - // read up to NROWS at a time - dvo_catalog_load_segment (&incatalog, VERBOSE, j*NROWS, NROWS); - - for (k = 0; k < incatalog.Nmeasure; k++) { - - averef = incatalog.measure[k].averef; - Ncat = outcat[averef]; - - Nout = outcatalog[Ncat].Nmeasure; - outcatalog[Ncat].measure[Nout] = incatalog.measure[k]; - outcatalog[Ncat].measure[Nout].averef = outref[averef]; - outcatalog[Ncat].Nmeasure++; - } - - for (n = 0; n < Noutcatalog; n++) { - dvo_catalog_save_segment (&outcatalog[N], VERBOSE, outcatalog[N].Nmeas_disk, outcatalog[N].Nmeasure); - } + // adjust depth + skylist[0].regions[i][0].table = FALSE; + for (j = 0; j < outlist[0].Nregions; j++) { + outlist[0].regions[j][0].table = TRUE; } - // split out the secfilt entries: - incatalog.catflags = LOAD_SECF; - Nblocks = incatalog.Nsecfilt_disk / NROWS; - for (j = 0; j < Nblocks; j++) { + } - // read up to NROWS at a time - dvo_catalog_load_segment (&incatalog, VERBOSE, j*NROWS, NROWS); + // save sky table copy + sprintf (filename, "%s/SkyTable.fits", CATDIR); + check_file_access (filename, FALSE, VERBOSE); + if (!SkyTableSave (sky, filename)) { + fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR); + exit (1); + } - for (k = 0; k < incatalog.Naverage; k++) { - - averef = j*NROWS + k; - Ncat = outcat[averef]; - Nout = outref[averef] * Nsecfilt; - - for (n = 0; n < Nsecfilt; n++) { - outcatalog[Ncat].secfilt[Nout + n] = incatalog.measure[k*Nsecfilt + n]; - } - } - - for (n = 0; n < Noutcatalog; n++) { - dvo_catalog_save_segment (&outcatalog[N], VERBOSE, outcatalog[N].Nmeas_disk, outcatalog[N].Nmeasure); - } - } - + exit (0); +} Index: trunk/Ohana/src/dvosplit/src/open_output_catalogs.c =================================================================== --- trunk/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/open_output_catalogs.c (revision 15743) @@ -0,0 +1,27 @@ +# include "dvosplit.h" + +Catalog *open_output_catalogs (SkyList *outlist) { + + int i; + Catalog *outcatalogs; + + ALLOCATE (outcatalogs, Catalog, outlist[0].Nregions); + + // an error exit status here is a significant error + for (i = 0; i < outlist[0].Nregions; i++) { + + // set the parameters which guide catalog open/load/create + outcatalogs[i].filename = outlist[0].filename[i]; + outcatalogs[i].Nsecfilt = GetPhotcodeNsecfilt (); + outcatalogs[i].catflags = LOAD_NONE; + outcatalogs[i].catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + outcatalogs[i].catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + + if (!dvo_catalog_open (&outcatalogs[i], outlist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open catalog file %s\n", outcatalogs[i].filename); + exit (2); + } + } + + return (outcatalogs); +} Index: trunk/Ohana/src/dvosplit/src/split_averages.c =================================================================== --- trunk/Ohana/src/dvosplit/src/split_averages.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/split_averages.c (revision 15743) @@ -0,0 +1,99 @@ +# include "dvosplit.h" +# define NROWS 100000 /* ~10MB per block for measures */ + +AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) { + + double inR, inD; + int n, block, ave, cat, averef, Nblocks, Ncat, Nout, Nsecfilt; + int *outref, *outcat; + AveLinks *avelinks; + + ALLOCATE (outref, int, incatalog[0].Naves_disk); + ALLOCATE (outcat, int, incatalog[0].Naves_disk); + + Nsecfilt = GetPhotcodeNsecfilt (); + + // allocate enough space for these output buffers: use Nsecfilt + 1 incase the file + // contains primary photcodes, which will increase Nsecfilt by one. + for (cat = 0; cat < outlist[0].Nregions; cat++) { + REALLOCATE (outcatalogs[cat].average, Average, NROWS); + REALLOCATE (outcatalogs[cat].secfilt, SecFilt, NROWS*(Nsecfilt + 1)); + } + + // split out the average & secfilt entries: + incatalog[0].catflags = LOAD_AVES | LOAD_SECF; + Nblocks = incatalog[0].Naves_disk / NROWS; + if (incatalog[0].Naves_disk % NROWS) Nblocks ++; + for (block = 0; block < Nblocks; block++) { + + // read up to NROWS at a time + dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); + assert (block*NROWS == incatalog[0].Naves_off); + + for (ave = 0; ave < incatalog[0].Naverage; ave++) { + averef = ave + incatalog[0].Naves_off; + + inR = incatalog[0].average[ave].R; + inD = incatalog[0].average[ave].D; + + // which of the outcatalogs contains this coordinate? + + Ncat = -1; + for (cat = 0; cat < outlist[0].Nregions; cat++) { + if (inR < outlist[0].regions[cat][0].Rmin) continue; + if (inR > outlist[0].regions[cat][0].Rmax) continue; + if (inD < outlist[0].regions[cat][0].Dmin) continue; + if (inD > outlist[0].regions[cat][0].Dmax) continue; + Ncat = cat; + break; + } + + if (Ncat == -1) { + fprintf (stderr, "WARNING: missed %d (%f, %f)\n", averef, inR, inD); + continue; + } + + Nout = outcatalogs[Ncat].Naverage; + outref[averef] = Nout + outcatalogs[Ncat].Naves_off; + outcat[averef] = Ncat; + + // assign the value to the next element of the output catalog + outcatalogs[Ncat].average[Nout] = incatalog[0].average[ave]; + outcatalogs[Ncat].Naverage ++; + + // update secfilt at the same time + for (n = 0; n < Nsecfilt; n++) { + outcatalogs[Ncat].secfilt[Nout*Nsecfilt + n] = incatalog[0].secfilt[ave*Nsecfilt + n]; + outcatalogs[Ncat].Nsecf_mem++; + } + } + + // double check the values of Naverage, Nsecf_mem? + + // write out the new values + for (cat = 0; cat < outlist[0].Nregions; cat++) { + outcatalogs[cat].catflags = LOAD_AVES | LOAD_SECF; + + fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt); + dvo_catalog_save (&outcatalogs[cat], VERBOSE); + fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt); + + // XXX I need to advance the pointers and free the current data + // XXX these should be done within save segment: + outcatalogs[cat].Naves_disk += outcatalogs[cat].Naverage; + outcatalogs[cat].Naves_off += outcatalogs[cat].Naverage; + outcatalogs[cat].Nsecf_disk += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; + outcatalogs[cat].Nsecf_off += outcatalogs[cat].Nsecfilt * outcatalogs[cat].Naverage; + outcatalogs[cat].Nsecfilt = Nsecfilt; + + outcatalogs[cat].Naverage = 0; + outcatalogs[cat].Nsecf_mem = 0; + } + } + + ALLOCATE (avelinks, AveLinks, 1); + avelinks[0].outref = outref; + avelinks[0].outcat = outcat; + + return (avelinks); +} Index: trunk/Ohana/src/dvosplit/src/split_measures.c =================================================================== --- trunk/Ohana/src/dvosplit/src/split_measures.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/split_measures.c (revision 15743) @@ -0,0 +1,48 @@ +# include "dvosplit.h" +# define NROWS 100000 /* ~10MB per row for measures */ + +int split_measures (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks) { + + int block, meas, cat, Nblocks, Ncat, Nout, averef; + int *outref, *outcat; + + outref = avelinks->outref; + outcat = avelinks->outcat; + + // allocate enough space for the output buffer + for (cat = 0; cat < outlist[0].Nregions; cat++) { + REALLOCATE (outcatalogs[cat].measure, Measure, NROWS); + } + + // split out the measure entries: + incatalog[0].catflags = LOAD_MEAS; + Nblocks = incatalog[0].Nmeas_disk / NROWS; + if (incatalog[0].Nmeas_disk % NROWS) Nblocks ++; + for (block = 0; block < Nblocks; block++) { + + // read up to NROWS at a time + dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); + + for (meas = 0; meas < incatalog[0].Nmeasure; meas++) { + + averef = incatalog[0].measure[meas].averef; + Ncat = outcat[averef]; + + Nout = outcatalogs[Ncat].Nmeasure; + outcatalogs[Ncat].measure[Nout] = incatalog[0].measure[meas]; + outcatalogs[Ncat].measure[Nout].averef = outref[averef]; + outcatalogs[Ncat].Nmeasure++; + } + + for (cat = 0; cat < outlist[0].Nregions; cat++) { + outcatalogs[cat].catflags = LOAD_MEAS; + dvo_catalog_save (&outcatalogs[cat], VERBOSE); + + outcatalogs[cat].Nmeas_disk += outcatalogs[cat].Nmeasure; + outcatalogs[cat].Nmeas_off += outcatalogs[cat].Nmeasure; + outcatalogs[cat].Nmeasure = 0; + + } + } + return (TRUE); +} Index: trunk/Ohana/src/dvosplit/src/split_missings.c =================================================================== --- trunk/Ohana/src/dvosplit/src/split_missings.c (revision 15743) +++ trunk/Ohana/src/dvosplit/src/split_missings.c (revision 15743) @@ -0,0 +1,47 @@ +# include "dvosplit.h" +# define NROWS 100000 /* ~10MB per row for missings */ + +int split_missings (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks) { + + int *outref, *outcat; + + outref = avelinks->outref; + outcat = avelinks->outcat; + + // allocate enough space for the output buffer + for (cat = 0; cat < outlist[0].Nregions; cat++) { + REALLOCATE (outcatalog[cat].missing, Missing, NROWS); + } + + // split out the missing entries: + incatalog[0].catflags = LOAD_MISS; + Nblocks = incatalog[0].Nmiss_disk / NROWS; + if (incatalog[0].Nmiss_disk % NROWS) Nblocks ++; + for (block = 0; block < Nblocks; block++) { + + // read up to NROWS at a time + dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS); + + for (miss = 0; miss < incatalog[0].Nmissing; miss++) { + + averef = incatalog[0].missing[miss].averef; + Ncat = outcat[averef]; + + Nout = outcatalog[Ncat].Nmissing; + outcatalog[Ncat].missing[Nout] = incatalog[0].missing[miss]; + outcatalog[Ncat].missing[Nout].averef = outref[averef]; + outcatalog[Ncat].Nmissing++; + } + + for (cat = 0; cat < outlist[0].Nregions; cat++) { + outcatalogs[cat].catflags = LOAD_MISS; + dvo_catalog_save_segment (&outcatalog[cat], VERBOSE); + + outcatalog[cat].Nmiss_disk += outcatalog[cat].Nmissing; + outcatalog[cat].Nmiss_off += outcatalog[cat].Nmissing; + outcatalog[cat].Nmissing = 0; + + } + } + return (TRUE); +}