Index: /trunk/Ohana/src/photdbc/Makefile =================================================================== --- /trunk/Ohana/src/photdbc/Makefile (revision 33654) +++ /trunk/Ohana/src/photdbc/Makefile (revision 33655) @@ -1,5 +1,5 @@ -default: photdbc +default: photdbc photdbc_client dvodist help: - @echo "make options: photdbc (default)" + @echo "make options: photdbc photdbc_client dvodist" include ../../Makefile.System @@ -18,8 +18,13 @@ photdbc: $(BIN)/photdbc.$(ARCH) -install: $(DESTBIN)/photdbc +photdbc_client: $(BIN)/photdbc_client.$(ARCH) + +dvodist: $(BIN)/dvodist.$(ARCH) + +install: $(DESTBIN)/photdbc $(DESTBIN)/photdbc_client $(DESTBIN)/dvodist PHOTDBC = \ $(SRC)/photdbc.$(ARCH).o \ +$(SRC)/photdbc_catalogs.$(ARCH).o \ $(SRC)/initialize.$(ARCH).o \ $(SRC)/ConfigInit.$(ARCH).o \ @@ -29,4 +34,32 @@ $(SRC)/join_stars.$(ARCH).o \ $(SRC)/make_subcatalog.$(ARCH).o + +PHOTDBC_CLIENT = \ +$(SRC)/photdbc_client.$(ARCH).o \ +$(SRC)/photdbc_catalogs.$(ARCH).o \ +$(SRC)/initialize.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/args.$(ARCH).o \ +$(SRC)/copy_images.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/join_stars.$(ARCH).o \ +$(SRC)/make_subcatalog.$(ARCH).o + +DVODIST = \ +$(SRC)/dvodist.$(ARCH).o \ +$(SRC)/initialize_dvodist.$(ARCH).o \ +$(SRC)/Shutdown_dvodist.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/md5_ops.$(ARCH).o \ +$(SRC)/md5.$(ARCH).o \ +$(SRC)/memstr.$(ARCH).o \ +$(SRC)/HostTableLoad.$(ARCH).o \ +$(SRC)/AssignSkyToHost.$(ARCH).o \ +$(SRC)/CopyBackupToHost.$(ARCH).o \ +$(SRC)/UseBackupForHost.$(ARCH).o \ +$(SRC)/CopyToHostLocation.$(ARCH).o \ +$(SRC)/CopyFromHostLocation.$(ARCH).o \ +$(SRC)/FixSkyForHost.$(ARCH).o \ +$(SRC)/PclientCommands.$(ARCH).o OLD = \ @@ -50,5 +83,10 @@ $(SRC)/wcatalog.$(ARCH).o - $(PHOTDBC): $(INC)/photdbc.h $(BIN)/photdbc.$(ARCH): $(PHOTDBC) + +$(PHOTDBC_CLIENT): $(INC)/photdbc.h +$(BIN)/photdbc_client.$(ARCH): $(PHOTDBC_CLIENT) + +$(DVODIST): $(INC)/dvodist.h +$(BIN)/dvodist.$(ARCH): $(DVODIST) Index: /trunk/Ohana/src/photdbc/include/dvodist.h =================================================================== --- /trunk/Ohana/src/photdbc/include/dvodist.h (revision 33655) +++ /trunk/Ohana/src/photdbc/include/dvodist.h (revision 33655) @@ -0,0 +1,61 @@ +# include +# include +# include +# include +# include // needed? + +# define PCLIENT_PROMPT "pclient:" + +// used by the md5 stuff +# define NDIGEST 16 + +typedef enum { + MODE_NONE = 0, + MODE_OUT = 1, + MODE_IN = 2, + MODE_FIX = 3, + MODE_OUT_BACKUP = 4, + MODE_USE_BACKUP = 5, +} ModeType; + +/* global variables */ +int VERBOSE; +SkyRegion UserPatch; +ModeType MODE; + +char *srcHostname; +char *dstHostname; + +# define MAX_PATH_LENGTH 1024 + +void usage(); +int args (int argc, char **argv); +void initialize (int argc, char **argv); + +void LockDatabase (char *catdir); +int CheckHostsAndPaths (HostTable *table); + +int AssignSkyToHost (SkyList *skylist, HostTable *table); +int CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table); +int CopyFromHostLocation(char *catdir, SkyList *skylist, HostTable *table); + +int Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2); +void lock_image_db (FITS_DB *db, char *filename); +void TrapSignal (int sig); +void SetProtect (int mode); +int SetSignals (void); + +int get_md5_with_copy (char *input, char *output, md5_byte_t *digest); +int hexchar_to_int (char input); +int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest); +int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest); + +int CheckBusyJob (HostInfo *host, IOBuffer *stdout_buf, IOBuffer *stderr_buf); +int GetJobOutput (char *command, HostInfo *host, IOBuffer *output, int size); +int PclientResponse (HostInfo *host, char *response, IOBuffer *buffer); +int PclientCommand (HostInfo *host, char *command, IOBuffer *buffer); + +int FixSkyForHost (char *catdir, SkyList *skylist, HostTable *table); + +int UseBackupForHost (char *catdir, SkyList *skylist, HostTable *table); +int CopyBackupToHost (char *catdir, SkyList *skylist, HostTable *table); Index: /trunk/Ohana/src/photdbc/include/md5.h =================================================================== --- /trunk/Ohana/src/photdbc/include/md5.h (revision 33655) +++ /trunk/Ohana/src/photdbc/include/md5.h (revision 33655) @@ -0,0 +1,94 @@ +/* + Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.h,v 1.1 2007-03-20 01:13:54 price Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.h is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2007-03-19 pap Fixing types for wider acceptance; incorporating into psLib. + 2002-04-13 lpd Removed support for non-ANSI compilers; removed + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); + added conditionalization for C++ compilation from Martin + Purschke . + 1999-05-03 lpd Original version. + */ + +#ifndef md5_INCLUDED +# define md5_INCLUDED + +/* + * This package supports both compile-time and run-time determination of CPU + * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be + * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is + * defined as non-zero, the code will be compiled to run only on big-endian + * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to + * run on either big- or little-endian CPUs, but will run slightly less + * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. + */ + +#include + +typedef uint8_t md5_byte_t; /* 8-bit byte */ +typedef uint32_t md5_word_t; /* 32-bit word */ + +/* Define the state of the MD5 Algorithm. */ +typedef struct md5_state_s { + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ +} md5_state_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* Initialize the algorithm. */ +void md5_init(md5_state_t *pms); + +/* Append a string to the message. */ +void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); + +/* Finish the message and return the digest. */ +void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* md5_INCLUDED */ Index: /trunk/Ohana/src/photdbc/include/photdbc.h =================================================================== --- /trunk/Ohana/src/photdbc/include/photdbc.h (revision 33654) +++ /trunk/Ohana/src/photdbc/include/photdbc.h (revision 33655) @@ -21,5 +21,14 @@ } StatType; +int PARALLEL; +int PARALLEL_MANUAL; +int PARALLEL_SERIAL; + +int HOST_ID; +char *HOSTDIR; + // need to get RADIUS from Config + +# define MAX_PATH_LENGTH 1024 /* global variables */ @@ -29,10 +38,10 @@ double UNIQ_RADIUS; double DMCAL_MIN;; -char ImageCat[256]; -char GSCFILE[256]; -char CATDIR[256]; -char CATMODE[16]; /* raw, mef, split, mysql */ -char CATFORMAT[16]; /* internal, elixir, loneos, panstarrs */ -char PhotCodeFile[256]; +char ImageCat[MAX_PATH_LENGTH]; +char GSCFILE[MAX_PATH_LENGTH]; +char CATDIR[MAX_PATH_LENGTH]; +char *CATMODE; /* raw, mef, split, mysql */ +char *CATFORMAT; /* internal, elixir, loneos, panstarrs */ +char PhotCodeFile[MAX_PATH_LENGTH]; double RMIN; @@ -124,2 +133,7 @@ int copy_images (char *outdir); void usage(); + +int photdbc_catalogs (char *outroot, SkyList *skylist, int hostID); +int photdbc_parallel (char *outroot, SkyList *skylist); +int args_client (int argc, char **argv); +void initialize_client (int argc, char **argv); Index: /trunk/Ohana/src/photdbc/src/AssignSkyToHost.c =================================================================== --- /trunk/Ohana/src/photdbc/src/AssignSkyToHost.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/AssignSkyToHost.c (revision 33655) @@ -0,0 +1,40 @@ +# include "dvodist.h" +# define DEBUG 1 + +int AssignSkyToHost (SkyList *skylist, HostTable *table) { + + int i; + long A, B; + + // init rnd seed + A = time(NULL); + for (B = 0; A == time(NULL); B++); + srand48(B); + + for (i = 0; i < skylist->Nregions; i++) { + + // assign or re-assign? if a region is already assigned, do we keep it? + + // XXX some options; +# if (0) + if (CLEAR_ERRORS && (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE)) { + skylist->regions[i]->hostID = 0; + } +# endif + + // do not reassign tables; this lets us run this program multiple times for different regions + // tables already assigned are left where they are + if (skylist->regions[i]->hostID) { + continue; + } + + // assign all regions to one of the Nhost hosts + float f = drand48(); + float n = table->Nhosts * f; + int N = MIN(table->Nhosts - 1, n); + skylist->regions[i]->hostID = table->hosts[N].hostID; + // fprintf (stderr, "%f %f %d %d\n", f, n, N, skylist->regions[i]->hostID); + } + + return (TRUE); +} Index: /trunk/Ohana/src/photdbc/src/ConfigInit.c =================================================================== --- /trunk/Ohana/src/photdbc/src/ConfigInit.c (revision 33654) +++ /trunk/Ohana/src/photdbc/src/ConfigInit.c (revision 33655) @@ -52,8 +52,12 @@ ScanConfig (config, "NMEAS_MIN_FILTERED", "%d", 0, &NMEAS_MIN_FILTERED); - WarnConfig (config, "GSCFILE", "%s", 0, GSCFILE); + // do not apply this limit for now + NCODE_MIN = 0; + WarnConfig (config, "CATDIR", "%s", 0, CATDIR); - ScanConfig (config, "CATMODE", "%s", 0, CATMODE); - ScanConfig (config, "CATFORMAT", "%s", 0, CATFORMAT); + char *tmpcatdir = abspath (CATDIR, MAX_PATH_LENGTH); + strcpy (CATDIR, tmpcatdir); + free (tmpcatdir); + ScanConfig (config, "PHOTCODE_FILE", "%s", 0, MasterPhotcodeFile); @@ -67,11 +71,8 @@ } - if (*CATMODE == 0) strcpy (CATMODE, "RAW"); - if (*CATFORMAT == 0) strcpy (CATFORMAT, "ELIXIR"); - /* XXX this does not yet write out the master photcode table */ sprintf (CatdirPhotcodeFile, "%s/Photcodes.dat", CATDIR); - if (!LoadPhotcodes (CatdirPhotcodeFile, MasterPhotcodeFile, TRUE)) { - fprintf (stderr, "error loading photcode table %s or master file %s\n", CatdirPhotcodeFile, MasterPhotcodeFile); + if (!LoadPhotcodes (CatdirPhotcodeFile, NULL, TRUE)) { + fprintf (stderr, "error loading photcode table %s\n", CatdirPhotcodeFile); exit (1); } Index: /trunk/Ohana/src/photdbc/src/CopyBackupToHost.c =================================================================== --- /trunk/Ohana/src/photdbc/src/CopyBackupToHost.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/CopyBackupToHost.c (revision 33655) @@ -0,0 +1,188 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// copy all *.bck currently on the srcHost to dstHost +int CopyBackupToHost (char *catdir, SkyList *skylist, HostTable *table) { + + int i, j, k; + struct stat filestat; + char srcname[1024]; + char tgtname[1024]; + char outdir[1024]; + + md5_byte_t srcDigest[NDIGEST]; + md5_byte_t tgtDigest[NDIGEST]; + + uid_t uid = getuid(); + gid_t gid = getgid(); + + // get the IDs for the src and dst machines + int srcID = 0; + int srcIndex = 0; + int dstID = 0; + int dstIndex = 0; + for (i = 0; i < table->Nhosts; i++) { + if (!strcmp (table->hosts[i].hostname, srcHostname)) { srcID = table->hosts[i].hostID; srcIndex = i; } + if (!strcmp (table->hosts[i].hostname, dstHostname)) { dstID = table->hosts[i].hostID; dstIndex = i; } + } + if (!srcID) { + fprintf (stderr, "failed to find source host %s\n", srcHostname); + exit (1); + } + if (!dstID) { + fprintf (stderr, "failed to find destination host %s\n", dstHostname); + exit (1); + } + if (srcID == dstID) { + fprintf (stderr, "invalid destination host %s : matches source host %s\n", dstHostname, srcHostname); + exit (1); + } + + HostInfo *dstHost = &table->hosts[dstIndex]; + + int Nfailure = 0; + for (i = 0; i < skylist->Nregions; i++) { + // skip unassigned tables + if (skylist->regions[i]->hostID != srcID) continue; + + // skip catalogs which have already been copied + if (skylist->regions[i]->hostFlags & DATA_ON_BCK) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // assign the dstHost to the backup ID + skylist->regions[i]->backupID = dstID; + + // we are copying CATDIR/*.bck to dstHost/pathname/* + + // have to succeed on all 4 tables; if we fail on any one, mark it as failed + int success = TRUE; + + char *subdir = pathname (skylist->regions[i]->name); + sprintf (outdir, "%s/%s", dstHost->pathname, subdir); + free (subdir); + + int status = check_dir_access (outdir, DEBUG); + if (!status) { + fprintf (stderr, "failed to find / create target path\n"); + exit (2); + } + + char extname[4][16] = {"cpm", "cpt", "cps", "cpn"}; + for (j = 0; success && (j < 4); j++) { + + // set the in and out table names + sprintf (srcname, "%s/%s.%s.bck", catdir, skylist->regions[i]->name, extname[j]); + sprintf (tgtname, "%s/%s.%s", dstHost->pathname, skylist->regions[i]->name, extname[j]); + + // does srcname exist & can it be read? + + // check permission to read file + status = stat (srcname, &filestat); + if (status) { + if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) + perror ("stat:"); + fprintf (stderr, "failure to access file %s\n", srcname); + success = FALSE; + continue; + } + + // can we read the file? + if (uid == filestat.st_uid) { + if (filestat.st_mode & S_IRUSR) goto valid; + } + if (gid == filestat.st_gid) { + if (filestat.st_mode & S_IRGRP) goto valid; + } + if (filestat.st_mode & S_IROTH) goto valid; + + fprintf (stderr, "cannot read file %s, skipping\n", srcname); + success = FALSE; + continue; + + valid: + if (j == 0) { + fprintf (stderr, "copying %s to %s\n", skylist->regions[i]->name, dstHost->pathname); + } + + // read srcname, write to tgtname, get MD5 sum for srcname as we go: + // XXX files that fail here should be skipped (but don't give up) + if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { + fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); + success = FALSE; + continue; + } + + // need to re-open and re-read tgtname to check md5sum + // XXX files that fail here need to be checked + // char *absname = abspath (tgtname, 1024); + // if (!get_md5_from_pclient (host, absname, tgtDigest)) { + if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { + fprintf (stderr, "error reading %s or getting md5\n", tgtname); + success = FALSE; + continue; + } + + // compare the two digest values + int match = TRUE; + for (k = 0; match && (k < NDIGEST); k++) { + match &= (tgtDigest[k] == srcDigest[k]); + } + + if (!match) { + if (DEBUG) { + fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname); + } + success = FALSE; + } + } + + if (!success) { + // if we failed to make a copy, remove the failure, mark the failure so we can retry + skylist->regions[i]->hostFlags |= DATA_COPY_FAILURE; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", dstHost->pathname, skylist->regions[i]->name, extname[j]); + if (!DEBUG) unlink (tgtname); + } + Nfailure ++; + } else { + skylist->regions[i]->hostFlags |= DATA_ON_BCK; + } + } + + if (!Nfailure) { + fprintf (stderr, "all tables successfully moved\n"); + } else { + fprintf (stderr, "%d tables were not moved\n", Nfailure); + } + + return (TRUE); +} + +/* state table for copy to/from location + + data on src (local catdir) : 00.hostID + data on tgt (remote catdir) : 10.hostID + failure to copy to tgt : 01.hostID + failure to copy to src : 11.hostID +*/ + +/*** + + hostID, backupID, and hostFlags: + + hostID gives the primary location of the data + backupID gives the second location of the data + + DATA_ON_TGT : 0x01 : file exists on hostID + DATA_ON_BCK : 0x02 : file exists on backupID + DATA_USE_BCK : 0x04 : read from backupID, not hostID + + DATA_COPY_FAILURE : error copying data to TGT (set for retry) + + + ***/ Index: /trunk/Ohana/src/photdbc/src/CopyFromHostLocation.c =================================================================== --- /trunk/Ohana/src/photdbc/src/CopyFromHostLocation.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/CopyFromHostLocation.c (revision 33655) @@ -0,0 +1,174 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// XXX collapse this and CopyToHostLocation (only difference is name of src/tgt and bits set on success) +int CopyFromHostLocation (char *catdir, SkyList *skylist, HostTable *table) { + + int i, j, k; + struct stat filestat; + char srcname[1024]; + char tgtname[1024]; + char outdir[1024]; + + md5_byte_t srcDigest[NDIGEST]; + md5_byte_t tgtDigest[NDIGEST]; + + uid_t uid = getuid(); + gid_t gid = getgid(); + + int Nfailure = 0; + for (i = 0; i < skylist->Nregions; i++) { + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + + // move each table from table->hosts[].pathname/name to CATDIR/name + + // skip unassigned tables + if (!skylist->regions[i]->hostID) continue; + + // skip catalogs which are not on remote location + if (!(skylist->regions[i]->hostFlags & DATA_ON_TGT)) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // find the host for the table + int realID = skylist->regions[i]->hostID; + short index = table->index[realID]; + HostInfo *host = &table->hosts[index]; + + // have to succeed on all 4 tables; if we fail on any one, + int success = TRUE; + + char *subdir = pathname (skylist->regions[i]->name); + sprintf (outdir, "%s/%s", host->pathname, subdir); + free (subdir); + + int status = check_dir_access (outdir, DEBUG); + if (!status) { + fprintf (stderr, "failed to find / create target path\n"); + exit (2); + } + + char extname[4][16] = {"cpm", "cpt", "cps", "cpn"}; + for (j = 0; success && (j < 4); j++) { + + // set the in and out table names + sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); + + // does srcname exist & can it be read? + + // check permission to read file + status = stat (srcname, &filestat); + if (status) { + if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) + perror ("stat:"); + fprintf (stderr, "failure to access file %s\n", srcname); + success = FALSE; + continue; + } + + // can we read the file? + if (uid == filestat.st_uid) { + if (filestat.st_mode & S_IRUSR) goto valid; + } + if (gid == filestat.st_gid) { + if (filestat.st_mode & S_IRGRP) goto valid; + } + if (filestat.st_mode & S_IROTH) goto valid; + + fprintf (stderr, "cannot read file %s, skipping\n", srcname); + success = FALSE; + continue; + + valid: + + // the output (tgtname) could exist and be a link to the same file; delete it if it exists + status = stat (tgtname, &filestat); + if (!status) { + // a tgtfile already exists; get rid of it + if (unlink (tgtname)) { + fprintf (stderr, "failed to remove CATDIR file %s\n", tgtname); + success = FALSE; + continue; + } + } + + // read srcname, write to tgtname, get MD5 sum for srcname as we go: + if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { + fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); + success = FALSE; + continue; + } + + // need to re-open and re-read tgtname to check md5sum + if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { + fprintf (stderr, "error reading %s or getting md5\n", tgtname); + success = FALSE; + continue; + } + + // compare the two digest values + int match = TRUE; + for (k = 0; match && (k < NDIGEST); k++) { + match &= (tgtDigest[k] == srcDigest[k]); + } + + if (!match) { + if (DEBUG) { + fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname); + } + success = FALSE; + } + } + + if (!success) { + // let's use the upper 2 bits of the hostID for informational purposes. this + // leaves us with up to 16383 hosts. + + // data on src = (hostID & 0x8000) >> 15 + // copy failed = (hostID & 0x4000) >> 14 + // if we failed to make a copy, + + skylist->regions[i]->hostFlags |= DATA_COPY_FAILURE; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); + if (!DEBUG) unlink (tgtname); + } + Nfailure ++; + } else { + skylist->regions[i]->hostFlags &= ~DATA_ON_TGT; + for (j = 0; success && (j < 4); j++) { + sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + if (!DEBUG && DELETE_ORIGINAL) unlink (srcname); + } + } + } + + if (!Nfailure) { + fprintf (stderr, "all tables successfully moved\n"); + } else { + fprintf (stderr, "%d tables were not moved\n", Nfailure); + } + + return (TRUE); +} + +/* state table for copy to/from location + + data on src (local catdir) : 00.hostID + data on tgt (remote catdir) : 10.hostID + failure to copy to tgt : 01.hostID + failure to copy to src : 11.hostID +*/ + +/* need to think a bit about behavior in the context of SPLIT vs MEF: + + * SPLIT : all 4 tables either exist or none do + * MEF only cpt table exists + + * read the SPLIT headers to see if MEASURE, MISSING, SECFILT fields exist? + */ Index: /trunk/Ohana/src/photdbc/src/CopyToHostLocation.c =================================================================== --- /trunk/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33655) @@ -0,0 +1,195 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// XXX collapse this and CopyFromHostLocation (only difference is name of src/tgt and bits set on success) +int CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table) { + + int i, j, k; + struct stat filestat; + char srcname[1024]; + char newname[1024]; + char tgtname[1024]; + char outdir[1024]; + + md5_byte_t srcDigest[NDIGEST]; + md5_byte_t tgtDigest[NDIGEST]; + + uid_t uid = getuid(); + gid_t gid = getgid(); + + int Nfailure = 0; + for (i = 0; i < skylist->Nregions; i++) { + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + + // move each table from CATDIR/name to table->hosts[].pathname/name + + // skip unassigned tables + if (!skylist->regions[i]->hostID) continue; + + // skip catalogs which have already been copied + if (skylist->regions[i]->hostFlags & DATA_ON_TGT) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // find the host for the table + int realID = skylist->regions[i]->hostID; + short index = table->index[realID]; + HostInfo *host = &table->hosts[index]; + + // have to succeed on all 4 tables; if we fail on any one, + int success = TRUE; + + char *subdir = pathname (skylist->regions[i]->name); + sprintf (outdir, "%s/%s", host->pathname, subdir); + free (subdir); + + int status = check_dir_access (outdir, DEBUG); + if (!status) { + fprintf (stderr, "failed to find / create target path\n"); + exit (2); + } + + char extname[4][16] = {"cpm", "cpt", "cps", "cpn"}; + for (j = 0; success && (j < 4); j++) { + + // set the in and out table names + sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); + sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + + // does srcname exist & can it be read? + + // check permission to read file + status = stat (srcname, &filestat); + if (status) { + if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) + perror ("stat:"); + fprintf (stderr, "failure to access file %s\n", srcname); + success = FALSE; + continue; + } + + // can we read the file? + if (uid == filestat.st_uid) { + if (filestat.st_mode & S_IRUSR) goto valid; + } + if (gid == filestat.st_gid) { + if (filestat.st_mode & S_IRGRP) goto valid; + } + if (filestat.st_mode & S_IROTH) goto valid; + + fprintf (stderr, "cannot read file %s, skipping\n", srcname); + success = FALSE; + continue; + + valid: + // read srcname, write to tgtname, get MD5 sum for srcname as we go: + // XXX files that fail here should be skipped (but don't give up) + if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { + fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); + success = FALSE; + continue; + } + + // need to re-open and re-read tgtname to check md5sum + // XXX files that fail here need to be checked + // char *absname = abspath (tgtname, 1024); + // if (!get_md5_from_pclient (host, absname, tgtDigest)) { + if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { + fprintf (stderr, "error reading %s or getting md5\n", tgtname); + success = FALSE; + continue; + } + + // compare the two digest values + int match = TRUE; + for (k = 0; match && (k < NDIGEST); k++) { + match &= (tgtDigest[k] == srcDigest[k]); + } + + if (!match) { + if (DEBUG) { + fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname); + } + success = FALSE; + } + } + + if (!success) { + // let's use the upper 2 bits of the hostID for informational purposes. this + // leaves us with up to 16383 hosts. + + // data on src = (hostID & 0x8000) >> 15 + // copy failed = (hostID & 0x4000) >> 14 + // if we failed to make a copy, + + skylist->regions[i]->hostFlags |= DATA_COPY_FAILURE; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + if (!DEBUG) unlink (tgtname); + } + Nfailure ++; + } else { + skylist->regions[i]->hostFlags |= DATA_ON_TGT; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); + + // do not rename or unlink original if it does not exist.. + status = stat (srcname, &filestat); + if (status && (errno == ENOENT)) continue; + + if (!DEBUG && DELETE_ORIGINAL) { + unlink (srcname); + } else { + sprintf (newname, "%s/%s.%s.bck", catdir, skylist->regions[i]->name, extname[j]); + status = stat (newname, &filestat); + if (!status) { + // a backup file already exists; get rid of it + if (unlink (newname)) { + fprintf (stderr, "failed to remove old backup file %s\n", newname); + continue; + } + } + if (rename (srcname, newname)) { + fprintf (stderr, "failed to rename %s to %s\n", srcname, newname); + continue; + } + char *absname = abspath (tgtname, 1024); + if (symlink (absname, srcname)) { + fprintf (stderr, "failed to make symlink from %s to %s\n", absname, srcname); + } + free (absname); + } + } + } + } + + if (!Nfailure) { + fprintf (stderr, "all tables successfully moved\n"); + } else { + fprintf (stderr, "%d tables were not moved\n", Nfailure); + } + + return (TRUE); +} + +/* state table for copy to/from location + + data on src (local catdir) : 00.hostID + data on tgt (remote catdir) : 10.hostID + failure to copy to tgt : 01.hostID + failure to copy to src : 11.hostID +*/ + +/* need to think a bit about behavior in the context of SPLIT vs MEF: + + * SPLIT : all 4 tables either exist or none do + * MEF only cpt table exists + + * read the SPLIT headers to see if MEASURE, MISSING, SECFILT fields exist? + */ + Index: /trunk/Ohana/src/photdbc/src/FixSkyForHost.c =================================================================== --- /trunk/Ohana/src/photdbc/src/FixSkyForHost.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/FixSkyForHost.c (revision 33655) @@ -0,0 +1,52 @@ +# include "dvodist.h" +# define DEBUG 0 + +// examine all of the filenames in the catdir directories and update the host table to note the correct locations +int FixSkyForHost (char *catdir, SkyList *skylist, HostTable *table) { + + int i, j; + + for (i = 0; i < skylist->Nregions; i++) { + + if (i && (i % 1000 == 0)) fprintf (stderr, "."); + + struct stat filestat; + + // filename of copy at CATDIR + char rawfile[1024]; + snprintf (rawfile, 1024, "%s/%s.cpt", catdir, skylist[0].regions[i]->name); + + int status = lstat (rawfile, &filestat); + if (status == -1) { + if (errno == ENOENT) continue; + fprintf (stderr, "failed to STAT file %s\n", rawfile); + continue; + } + + if (!S_ISLNK(filestat.st_mode)) continue; + + char linkfile[1024]; + ssize_t Nbytes = readlink (rawfile, linkfile, 1024); + if (Nbytes == -1) { + fprintf (stderr, "error reading link %s\n", rawfile); + continue; + } + + int found = FALSE; + for (j = 0; !found && (j < table->Nhosts); j++) { + if (strncmp (linkfile, table->hosts[j].pathname, strlen(table->hosts[j].pathname))) continue; + // found the host + HostInfo *host = &table->hosts[j]; + skylist->regions[i]->hostID = host->hostID; + skylist->regions[i]->hostFlags |= DATA_ON_TGT; + found = TRUE; + + if (DEBUG) fprintf (stderr, "found %s on %s\n", rawfile, host->hostname); + } + + if (!found) { + fprintf (stderr, "failed to find host for %s (%s)\n", rawfile, linkfile); + } + } + return (TRUE); +} Index: /trunk/Ohana/src/photdbc/src/HostTableLoad.c =================================================================== --- /trunk/Ohana/src/photdbc/src/HostTableLoad.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/HostTableLoad.c (revision 33655) @@ -0,0 +1,37 @@ +# include "dvodist.h" +# define DEBUG 1 + +int CheckHostsAndPaths (HostTable *table) { + + int i; + char command[64]; + char shell[64]; + + strcpy (command, "ssh"); + strcpy (shell, "pclient"); + + for (i = 0; i < table->Nhosts; i++) { + + // verify that I can ssh to this machine + if (DEBUG) fprintf (stderr, "starting host within thread\n"); + + int errorInfo; + int pid = rconnect (command, table->hosts[i].hostname, shell, table->hosts[i].stdio, &errorInfo, TRUE); + if (!pid) { + /** failure to start: extend retry period **/ + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; + // keep the connection open to use for md5sum check + + int status = check_dir_access (table->hosts[i].pathname, DEBUG); + if (!status) { + fprintf (stderr, "failed to find / create target path\n"); + exit (2); + } + + fprintf (stderr, "success : %s, %s\n", table->hosts[i].hostname, table->hosts[i].pathname); + } + return (TRUE); +} Index: /trunk/Ohana/src/photdbc/src/PclientCommands.c =================================================================== --- /trunk/Ohana/src/photdbc/src/PclientCommands.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/PclientCommands.c (revision 33655) @@ -0,0 +1,208 @@ +# include "dvodist.h" +# define PCLIENT_TIMEOUT 1000 +# define DEBUG 0 + +// send a command and check for errors; ignore output +int PclientCommand (HostInfo *host, char *command, IOBuffer *buffer) { + + int status; + + // flush the stdout and stderr buffers here + ReadtoIOBuffer (buffer, host->stdio[1]); + FlushIOBuffer (buffer); + ReadtoIOBuffer (buffer, host->stdio[2]); + FlushIOBuffer (buffer); + + /* send command to client (adding on \n) */ + status = write_fmt (host->stdio[0], "%s\n", command); + + /* is pipe still open? */ + if ((status == -1) && (errno == EPIPE)) { + fprintf (stderr, "pclient read gives pipe error for %s\n", command); + return FALSE; + } + + FlushIOBuffer (buffer); + + return TRUE; +} + +// check for response; message must end with specified string. +// accumulate the response in the buffer +int PclientResponse (HostInfo *host, char *response, IOBuffer *buffer) { + + int i; + int status; + char *line; + struct timespec request, remain; + + /* avoid blocking very long on read, test every 100 usec, up to 0.1 sec */ + request.tv_sec = 0; + request.tv_nsec = 100000; + + /* watch for response - wait up to 1 second */ + line = NULL; + status = -1; + + // how long does each cycle really take? + for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) { + status = ReadtoIOBuffer (buffer, host->stdio[1]); + line = memstr (buffer->buffer, response, buffer->Nbuffer); + if (status == -1) nanosleep (&request, &remain); + } + if (status == 0) { + fprintf (stderr, "pclient read returns 0 for %s\n", response); + return FALSE; + } + if (!line) { + fprintf (stderr, "client hung\n"); + return FALSE; + } + if (status == -1) { + fprintf (stderr, "client hung\n"); + return FALSE; + } + + // fprintf (stderr, "response: %s\n", buffer->buffer); + + return TRUE; +} + +int GetJobOutput (char *command, HostInfo *host, IOBuffer *output, int size) { + + char *line; + int i, status; + struct timespec request, remain; + + /* avoid blocking on waitpid, test every 100 usec, up to 10 msec */ + request.tv_sec = 0; + request.tv_nsec = 100000; + + status = write_fmt (host->stdio[0], "%s\n", command); + + if ((status == -1) && (errno == EPIPE)) { + fprintf (stderr, "host crashed?\n"); + goto escape; + } + + // attempt to read the output->size bytes from the host + for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (output->Nbuffer < size); i++) { + status = ReadtoIOBuffer (output, host->stdio[1]); + if (status == -1) nanosleep (&request, &remain); + } + if (DEBUG) fprintf (stderr, "Read %d of %d bytes so far\n", output->Nbuffer, size); + if (status == 0) { + fprintf (stderr, "client down\n"); + goto escape; + } + if (output->Nbuffer < size) { + fprintf (stderr, "client hung\n"); + goto escape; + } + + // keep trying to read until we get the prompt + line = NULL; + status = -1; + for (i = 0; (i < PCLIENT_TIMEOUT) && (status != 0) && (line == NULL); i++) { + status = ReadtoIOBuffer (output, host->stdio[1]); + line = memstr (output->buffer, PCLIENT_PROMPT, output->Nbuffer); + if (status == -1) nanosleep (&request, &remain); + } + if (DEBUG) fprintf (stderr, "Read %d of %d bytes so far\n", output->Nbuffer, size); + if (!status) { + fprintf (stderr, "pclient down?\n"); // return PCLIENT_DOWN; + goto escape; + } + if (!line) { + fprintf (stderr, "pclient hung?\n"); // return PCLIENT_HUNG; + goto escape; + } + return TRUE; + +escape: + return FALSE; +} + +int CheckBusyJob (HostInfo *host, IOBuffer *stdout_buf, IOBuffer *stderr_buf) { + + int status; + char *p; + char string[64]; + IOBuffer buffer; + + InitIOBuffer (&buffer, 100); + status = PclientCommand (host, "status", &buffer); + status = PclientResponse (host, PCLIENT_PROMPT, &buffer); + + /** need to parse message **/ + p = memstr (buffer.buffer, "STATUS", buffer.Nbuffer); + if (p == NULL) { + if (DEBUG) fprintf (stderr, "missing STATUS in response; try again?\n"); + FreeIOBuffer (&buffer); + return FALSE; + } + + sscanf (p, "%*s %s", string); + if (!strcmp(string, "NONE")) { + if (DEBUG) fprintf (stderr, "not started?\n"); + FreeIOBuffer (&buffer); + return FALSE; + } + + /** no status change, return to BUSY stack **/ + if (!strcmp(string, "BUSY")) { + if (DEBUG) fprintf (stderr, "not yet done...\n"); + FreeIOBuffer (&buffer); + return FALSE; + } + + /* exit status better be either EXIT or CRASH */ + if (!strcmp(string, "CRASH")) { + fprintf (stderr, "crash on host...\n"); + FreeIOBuffer (&buffer); + return FALSE; + } + + if (strcmp(string, "EXIT")) { + fprintf (stderr, "error in status string?\n"); + FreeIOBuffer (&buffer); + return FALSE; + } + + int exit_status, stdout_buf_size, stderr_buf_size; + + /* parse the exit status and sizes of output buffers */ + p = memstr (buffer.buffer, "EXITST", buffer.Nbuffer); + sscanf (p, "%*s %d", &exit_status); + p = memstr (buffer.buffer, "STDOUT", buffer.Nbuffer); + sscanf (p, "%*s %d", &stdout_buf_size); + p = memstr (buffer.buffer, "STDERR", buffer.Nbuffer); + sscanf (p, "%*s %d", &stderr_buf_size); + + // XXX runaway job if output too large? + if (stdout_buf_size > 0x1000000) abort(); + if (stderr_buf_size > 0x1000000) abort(); + + if (stdout_buf_size) { + GetJobOutput ("stdout", host, stdout_buf, stdout_buf_size); + } else { + ReadtoIOBuffer (&buffer, host->stdio[1]); + } + if (stderr_buf_size) { + GetJobOutput ("stderr", host, stderr_buf, stderr_buf_size); + } else { + ReadtoIOBuffer (&buffer, host->stdio[2]); + } + + FreeIOBuffer (&buffer); + return TRUE; +} + +/* memstr returns a view, not an allocated string : don't free */ +/* ReadtoIOBuffer returns : + 0 - pipe closed + -1 - no more data in pipe, data not ready + -2 - serious error reading from pipe + >0 - data read from pipe +*/ + Index: /trunk/Ohana/src/photdbc/src/Shutdown_dvodist.c =================================================================== --- /trunk/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 33655) @@ -0,0 +1,49 @@ +# include "dvodist.h" + +static FITS_DB db; + +void LockDatabase (char *catdir) { + + char ImageCat[1024]; + + // lock image table + snprintf (ImageCat, 1024, "%s/Images.dat", catdir); + + // lock the image db table + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD); + if (!status) { + Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + } + + // if the file is missing, the function above will succeed, but the db.dbstate will have + // a value of either: LCK_EMPTY or LCK_MISSING + if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) { + Shutdown ("no images?"); + } + + // XXX this program should be able to operate on a dvo database without any images (like + // 2MASS). but, we still need to lock the database when moving the files around to + // avoid collisions. So, we probably need to create an empty image table and lock it if + // there are no images at this time. +} + +/* clean up open / locked ImageCat before shutting down */ +int Shutdown (char *format, ...) { + va_list argp; + char *formatplus; + + ALLOCATE (formatplus, char, strlen(format)); + strcpy (formatplus, format); + strcat (formatplus, "\n"); + + va_start (argp, format); + vfprintf (stderr, formatplus, argp); + free (formatplus); + va_end (argp); + + // XXX this is in SetSignals.c, which includes photdbc.h + SetProtect (TRUE); + gfits_db_close (&db); + fprintf (stderr, "ERROR: dvodist halted\n"); + exit (1); +} Index: /trunk/Ohana/src/photdbc/src/UseBackupForHost.c =================================================================== --- /trunk/Ohana/src/photdbc/src/UseBackupForHost.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/UseBackupForHost.c (revision 33655) @@ -0,0 +1,57 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// set DATA_USE_BCK for all tables with DATA_ON_BCK and host matches srcHost +int UseBackupForHost (char *catdir, SkyList *skylist, HostTable *table) { + + int i; + int srcID, srcIndex; + + // get the IDs for the src and dst machines + srcID = srcIndex = 0; + for (i = 0; i < table->Nhosts; i++) { + if (!strcmp (table->hosts[i].hostname, srcHostname)) { + srcID = table->hosts[i].hostID; + srcIndex = i; + } + } + if (!srcID) { + fprintf (stderr, "failed to find source host %s\n", srcHostname); + exit (1); + } + + for (i = 0; i < skylist->Nregions; i++) { + // skip unassigned tables + if (skylist->regions[i]->hostID != srcID) continue; + + // skip catalogs which have not been copied + if (!(skylist->regions[i]->hostFlags & DATA_ON_BCK)) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // set the USE_BCK flag + skylist->regions[i]->hostFlags |= DATA_USE_BCK; + } + + return (TRUE); +} + +/*** + + hostID, backupID, and hostFlags: + + hostID gives the primary location of the data + backupID gives the second location of the data + + DATA_ON_TGT : 0x01 : file exists on hostID + DATA_ON_BCK : 0x02 : file exists on backupID + DATA_USE_BCK : 0x04 : read from backupID, not hostID + + DATA_COPY_FAILURE : error copying data to TGT (set for retry) + + + ***/ Index: /trunk/Ohana/src/photdbc/src/args.c =================================================================== --- /trunk/Ohana/src/photdbc/src/args.c (revision 33654) +++ /trunk/Ohana/src/photdbc/src/args.c (revision 33655) @@ -40,4 +40,20 @@ remove_argument (N, &argc, argv); MAX_MIN_MAG = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog format (PS1_V1, PS1_REF, etc) + CATFORMAT = NULL; + if ((N = get_argument (argc, argv, "-set-format"))) { + remove_argument (N, &argc, argv); + CATFORMAT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog mode (raw, mef, split, mysql) + CATMODE = NULL; + if ((N = get_argument (argc, argv, "-set-mode"))) { + remove_argument (N, &argc, argv); + CATMODE = strcreate (argv[N]); remove_argument (N, &argc, argv); } @@ -80,4 +96,32 @@ remove_argument (N, &argc, argv); PHOTCODE_SKIP_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + + // XXX for the moment, make this selection manual. it needs to be automatic + // based on the state of the SkyTable + PARALLEL = FALSE; + if ((N = get_argument (argc, argv, "-parallel"))) { + PARALLEL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the remote jobs and waiting for completion, + // relphot will simply list the remote command and wait for the user to signal completion + PARALLEL_MANUAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-manual"))) { + PARALLEL = TRUE; // -parallel-manual implies -parallel + PARALLEL_MANUAL = TRUE; + remove_argument (N, &argc, argv); + } + // this is a test mode : rather than launching the relphot_client jobs remotely, they are + // run in serial via 'system' + PARALLEL_SERIAL = FALSE; + if ((N = get_argument (argc, argv, "-parallel-serial"))) { + if (PARALLEL_MANUAL) { + fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n"); + exit (1); + } + PARALLEL = TRUE; // -parallel-serial implies -parallel + PARALLEL_SERIAL = TRUE; remove_argument (N, &argc, argv); } @@ -99,4 +143,138 @@ } +int args_client (int argc, char **argv) { + + int N; + + PARALLEL = FALSE; + PARALLEL_MANUAL = FALSE; + PARALLEL_SERIAL = FALSE; + + HOST_ID = 0; + if ((N = get_argument (argc, argv, "-hostID"))) { + remove_argument (N, &argc, argv); + HOST_ID = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOST_ID) usage(); + + HOSTDIR = NULL; + if ((N = get_argument (argc, argv, "-hostdir"))) { + remove_argument (N, &argc, argv); + HOSTDIR = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!HOSTDIR) usage(); + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + SHOW_PARAMS = FALSE; + if ((N = get_argument (argc, argv, "-params"))) { + remove_argument (N, &argc, argv); + SHOW_PARAMS = TRUE; + } + + ExcludeByInstMag = FALSE; + if ((N = get_argument (argc, argv, "-instmag"))) { + ExcludeByInstMag = TRUE; + remove_argument (N, &argc, argv); + INST_MAG_MIN = atof(argv[N]); + remove_argument (N, &argc, argv); + INST_MAG_MAX = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + ExcludeByMinSigma = FALSE; + if ((N = get_argument (argc, argv, "-min-sigma"))) { + ExcludeByMinSigma = TRUE; + remove_argument (N, &argc, argv); + SIGMA_MIN_KEEP = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + ExcludeByMaxMinMag = FALSE; + if ((N = get_argument (argc, argv, "-maxminmag"))) { + ExcludeByMaxMinMag = TRUE; + remove_argument (N, &argc, argv); + MAX_MIN_MAG = atof(argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog format (PS1_V1, PS1_REF, etc) + CATFORMAT = NULL; + if ((N = get_argument (argc, argv, "-set-format"))) { + remove_argument (N, &argc, argv); + CATFORMAT = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + // override input catalog mode (raw, mef, split, mysql) + CATMODE = NULL; + if ((N = get_argument (argc, argv, "-set-mode"))) { + remove_argument (N, &argc, argv); + CATMODE = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + + /* specify portion of the sky */ + REGION.Rmin = 0; + REGION.Rmax = 360; + REGION.Dmin = -90; + REGION.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + REGION.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + REGION.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + REGION.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + REGION.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + + if (REGION.Rmin == REGION.Rmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n"); + exit (2); + } + if (REGION.Dmin == REGION.Dmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n"); + exit (2); + } + } + + PHOTCODE_DROP_LIST = NULL; + if ((N = get_argument (argc, argv, "-photcode-drop"))) { + remove_argument (N, &argc, argv); + PHOTCODE_DROP_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + + PHOTCODE_SKIP_LIST = NULL; + if ((N = get_argument (argc, argv, "-photcode-skip"))) { + remove_argument (N, &argc, argv); + PHOTCODE_SKIP_LIST = strcreate(argv[N]); + remove_argument (N, &argc, argv); + } + + if (argc != 2) usage(); + + if ((REGION.Rmin == 0) && (REGION.Rmax == 360) && (REGION.Dmin == -90) && (REGION.Dmax == +90)) { + int i; + fprintf (stderr, "you have requested a copy of the entire sky in one pass\n"); + fprintf (stderr, "this could be a time consuming operation. type Ctrl-C within 5 seconds to cancel\n"); + for (i = 5; i > 0; i--) { + fprintf (stderr, "%d.. ", i); + usleep (1000000); + } + fprintf (stderr, "\n"); + } + + return (TRUE); +} + void usage() { Index: /trunk/Ohana/src/photdbc/src/copy_images.c =================================================================== --- /trunk/Ohana/src/photdbc/src/copy_images.c (revision 33654) +++ /trunk/Ohana/src/photdbc/src/copy_images.c (revision 33655) @@ -41,6 +41,6 @@ if (out.dbstate != LCK_EMPTY) Shutdown ("ERROR: image table exists %s", ImageOut); - out.mode = dvo_catalog_catmode (CATMODE); - out.format = dvo_catalog_catformat (CATFORMAT); + out.mode = CATMODE ? dvo_catalog_catmode (CATMODE) : in.mode; + out.format = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : in.format; dvo_image_create (&out, ZERO_POINT); Index: /trunk/Ohana/src/photdbc/src/dvodist.c =================================================================== --- /trunk/Ohana/src/photdbc/src/dvodist.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/dvodist.c (revision 33655) @@ -0,0 +1,60 @@ +# include "dvodist.h" + +// dvodist (-out | -in) (catdir) +// dvodist -out : copy catalog tables to distributed locations in host table (from catdir) +// dvodist -in : copy catalog tables to catdir (from distributed locations in host table) + +int main (int argc, char **argv) { + + SkyTable *sky; + SkyList *skylist; + + /* get configuration info, args, lockfile */ + SetSignals (); + initialize (argc, argv); + char *catdir = strcreate (argv[1]); + + // LockDatabase (catdir); + + // load the current SkyTable. If SkyTable does not exist, we must fail + sky = SkyTableLoadOptimal (catdir, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE); + SkyTableSetFilenames (sky, catdir, "cpt"); + skylist = SkyListByPatch (sky, -1, &UserPatch); + + HostTable *hosts = HostTableLoad (catdir, sky->hosts); + if (!hosts) { + Shutdown ("failed to load Host Table %s for %s\n", catdir, sky->hosts); + } + + // XXX I would like to do something so a big dvodist operation is easy to recover + // XXX BUT, I cannot save the table here without some caution (byteswap to and fro) + + switch (MODE) { + case MODE_OUT: + CheckHostsAndPaths(hosts); + AssignSkyToHost (skylist, hosts); + CopyToHostLocation (catdir, skylist, hosts); + break; + case MODE_IN: + CheckHostsAndPaths(hosts); + CopyFromHostLocation (catdir, skylist, hosts); + break; + case MODE_FIX: + FixSkyForHost (catdir, skylist, hosts); + break; + case MODE_OUT_BACKUP: + CopyBackupToHost (catdir, skylist, hosts); + break; + case MODE_USE_BACKUP: + UseBackupForHost (catdir, skylist, hosts); + break; + default: + fprintf (stderr, "impossible!"); + abort(); + } + + char *skyfile = SkyTableFilename (catdir); + SkyTableSave (sky, skyfile); + + exit (0); +} Index: /trunk/Ohana/src/photdbc/src/initialize.c =================================================================== --- /trunk/Ohana/src/photdbc/src/initialize.c (revision 33654) +++ /trunk/Ohana/src/photdbc/src/initialize.c (revision 33655) @@ -2,7 +2,4 @@ void initialize (int argc, char **argv) { - - int NPHOTCODES; - char *codename, *ptr, *list; /* are these set correctly? */ @@ -15,45 +12,6 @@ args (argc, argv); - NphotcodesDrop = 0; - photcodesDrop = NULL; - if (PHOTCODE_DROP_LIST != NULL) { - NPHOTCODES = 10; - ALLOCATE (photcodesDrop, PhotCode *, NPHOTCODES); - - /* parse the comma-separated list of photcodesDrop */ - list = PHOTCODE_DROP_LIST; - while ((codename = strtok_r (list, ",", &ptr)) != NULL) { - list = NULL; // pass NULL on successive strtok_r calls - fprintf (stderr, "PHOTCODE_LIST: %s\n", PHOTCODE_DROP_LIST); - fprintf (stderr, "codename: %s\n", codename); - if ((photcodesDrop[NphotcodesDrop] = GetPhotcodebyName (codename)) == NULL) { - fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); - exit (1); - } - NphotcodesDrop ++; - CHECK_REALLOCATE (photcodesDrop, PhotCode *, NPHOTCODES, NphotcodesDrop, 10); - } - } - - NphotcodesSkip = 0; - photcodesSkip = NULL; - if (PHOTCODE_SKIP_LIST != NULL) { - NPHOTCODES = 10; - ALLOCATE (photcodesSkip, PhotCode *, NPHOTCODES); - - /* parse the comma-separated list of photcodesSkip */ - list = PHOTCODE_SKIP_LIST; - while ((codename = strtok_r (list, ",", &ptr)) != NULL) { - list = NULL; // pass NULL on successive strtok_r calls - fprintf (stderr, "PHOTCODE_LIST: %s\n", PHOTCODE_SKIP_LIST); - fprintf (stderr, "codename: %s\n", codename); - if ((photcodesSkip[NphotcodesSkip] = GetPhotcodebyName (codename)) == NULL) { - fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); - exit (1); - } - NphotcodesSkip ++; - CHECK_REALLOCATE (photcodesSkip, PhotCode *, NPHOTCODES, NphotcodesSkip, 10); - } - } + photcodesDrop = ParsePhotcodeList (PHOTCODE_DROP_LIST, &NphotcodesDrop, FALSE); + photcodesSkip = ParsePhotcodeList (PHOTCODE_SKIP_LIST, &NphotcodesSkip, FALSE); if (SHOW_PARAMS) { @@ -84,2 +42,42 @@ } +void initialize_client (int argc, char **argv) { + + /* are these set correctly? */ + if (get_argument (argc, argv, "-h")) usage(); + if (get_argument (argc, argv, "--h")) usage(); + if (get_argument (argc, argv, "-help")) usage(); + if (get_argument (argc, argv, "--help")) usage(); + + ConfigInit (&argc, argv); + args_client (argc, argv); + + photcodesDrop = ParsePhotcodeList (PHOTCODE_DROP_LIST, &NphotcodesDrop, FALSE); + photcodesSkip = ParsePhotcodeList (PHOTCODE_SKIP_LIST, &NphotcodesSkip, FALSE); + + if (SHOW_PARAMS) { + fprintf (stderr, "current parameter settings:\n"); + fprintf (stderr, "VERBOSE: %d\n", VERBOSE); + fprintf (stderr, "JOIN_RADIUS: %lf\n", JOIN_RADIUS); + fprintf (stderr, "UNIQ_RADIUS: %lf\n", UNIQ_RADIUS); + + fprintf (stderr, "XMIN: %lf\n", XMIN); + fprintf (stderr, "XMAX: %lf\n", XMAX); + fprintf (stderr, "YMIN: %lf\n", YMIN); + fprintf (stderr, "YMAX: %lf\n", YMAX); + fprintf (stderr, "MMIN: %lf\n", MMIN); + fprintf (stderr, "MMAX: %lf\n", MMAX); + fprintf (stderr, "DMCAL_MIN: %lf\n", DMCAL_MIN); + fprintf (stderr, "DMSYS: %lf\n", DMSYS); + + fprintf (stderr, "CHISQ_MAX: %lf\n", CHISQ_MAX); + fprintf (stderr, "NMEAS_MIN: %d\n", NMEAS_MIN); + + fprintf (stderr, "IMAGE_CATALOG %s\n", ImageCat); + fprintf (stderr, "GSCFILE %s\n", GSCFILE); + fprintf (stderr, "CATDIR %s\n", CATDIR); + fprintf (stderr, "PHOTCODE_FILE %s\n", PhotCodeFile); + + exit (0); + } +} Index: /trunk/Ohana/src/photdbc/src/initialize_dvodist.c =================================================================== --- /trunk/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33655) @@ -0,0 +1,116 @@ +# include "dvodist.h" + +void initialize (int argc, char **argv) { + + /* are these set correctly? */ + if (get_argument (argc, argv, "-h")) usage(); + if (get_argument (argc, argv, "--h")) usage(); + if (get_argument (argc, argv, "-help")) usage(); + if (get_argument (argc, argv, "--help")) usage(); + + args (argc, argv); +} + +void usage() { + + fprintf (stderr, "USAGE: dvodist (-out | -in) (catdir)\n\n"); + fprintf (stderr, " this program takes an existing DVO database and distributes it to the remote machine locations\n"); + + fprintf (stderr, " optional options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -params : list the current parameters\n\n"); + + fprintf (stderr, " -out-backup (host1) to (host2) : move *.bck files for host1 to host2\n"); + fprintf (stderr, " -use-backup (host1) : use the backup file for host1\n"); + + exit (2); +} + +int args (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + /* specify portion of the sky */ + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + + if (UserPatch.Rmin == UserPatch.Rmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n"); + exit (2); + } + if (UserPatch.Dmin == UserPatch.Dmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n"); + exit (2); + } + } + + MODE = MODE_NONE; + if ((N = get_argument (argc, argv, "-in"))) { + MODE = MODE_IN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-out"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use both -in and -out options!\n"); + usage(); + } + MODE = MODE_OUT; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-fix"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use -fix with -in or -out options!\n"); + usage(); + } + MODE = MODE_FIX; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-out-backup"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use -fix with -in or -out options!\n"); + usage(); + } + // usage dvodist -out-backup (srcHost) to (dstHost) + MODE = MODE_OUT_BACKUP; + remove_argument (N, &argc, argv); + srcHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + remove_argument (N, &argc, argv); // remove 'to' + dstHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-use-backup"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use -fix with -in or -out options!\n"); + usage(); + } + // usage dvodist -out-backup (srcHost) to (dstHost) + MODE = MODE_USE_BACKUP; + remove_argument (N, &argc, argv); + srcHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if (!MODE) usage(); + + if (argc != 2) usage(); + + return (TRUE); +} Index: /trunk/Ohana/src/photdbc/src/md5.c =================================================================== --- /trunk/Ohana/src/photdbc/src/md5.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/md5.c (revision 33655) @@ -0,0 +1,386 @@ +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.1 2007-03-20 01:13:54 price Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + . Other authors are noted in the change history + that follows (in reverse chronological order): + + 2007-03-19 pap Added support for autoconf; incorporating into psLib. + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "md5.h" +#include + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef WORDS_BIGENDIAN // Comes from autoconf +# define BYTE_ORDER (WORDS_BIGENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((md5_word_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + + +static void +md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) +{ + md5_word_t + a = pms->abcd[0], b = pms->abcd[1], + c = pms->abcd[2], d = pms->abcd[3]; + md5_word_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + md5_word_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + md5_word_t xbuf[16]; + const md5_word_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const md5_byte_t *)0) & 3)) { + /* data are properly aligned */ + X = (const md5_word_t *)data; + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const md5_byte_t *xp = data; + int i; + +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + pms->abcd[0] += a; + pms->abcd[1] += b; + pms->abcd[2] += c; + pms->abcd[3] += d; +} + +void +md5_init(md5_state_t *pms) +{ + pms->count[0] = pms->count[1] = 0; + pms->abcd[0] = 0x67452301; + pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + pms->abcd[3] = 0x10325476; +} + +void +md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) +{ + const md5_byte_t *p = data; + int left = nbytes; + int offset = (pms->count[0] >> 3) & 63; + md5_word_t nbits = (md5_word_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + pms->count[1] += nbytes >> 29; + pms->count[0] += nbits; + if (pms->count[0] < nbits) + pms->count[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(pms->buf + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(pms, pms->buf); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(pms, p); + + /* Process a final partial block. */ + if (left) + memcpy(pms->buf, p, left); +} + +void +md5_finish(md5_state_t *pms, md5_byte_t digest[16]) +{ + static const md5_byte_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + md5_byte_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_append(pms, data, 8); + for (i = 0; i < 16; ++i) + digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); +} Index: /trunk/Ohana/src/photdbc/src/md5_ops.c =================================================================== --- /trunk/Ohana/src/photdbc/src/md5_ops.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/md5_ops.c (revision 33655) @@ -0,0 +1,185 @@ +# include "dvodist.h" + +// these are md5 operations that could probably be moved to libohana + +# define NBUFFER 512 + +// read a file, get the md5 sum, optionally write to 'output' +int get_md5_with_copy (char *input, char *output, md5_byte_t *digest) { + + // open the src file + FILE *fInput = fopen (input, "r"); + int fdInput = fileno (fInput); + + // open the tgt file + FILE *fOutput = NULL; + int fdOutput = 0; + if (output) { + fOutput = fopen (output, "w"); + fdOutput = fileno (fOutput); + } + + int nbytes; + md5_state_t state; + md5_byte_t buffer[NBUFFER]; + + md5_init (&state); + while ((nbytes = read (fdInput, buffer, NBUFFER)) > 0) { + md5_append (&state, buffer, nbytes); + if (output) { + int nbytesOut = write (fdOutput, buffer, nbytes); + if (nbytesOut != nbytes) { + return FALSE; + } + } + } + md5_finish (&state, digest); + +# if (0) + int i; + for (i = 0; i < NDIGEST; i++) { + fprintf (stderr, "%02x", digest[i]); + } + fprintf (stderr, " : %s\n", input); +# endif + + fclose (fInput); + fflush (fInput); + + if (output) { + fclose (fOutput); + fflush (fOutput); + } + return TRUE; +} + +// XXX this would be better controlled if we used a remote pclient. then we could +// check things like the exit status and +int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest) { + + IOBuffer buffer; + + char line[MAX_PATH_LENGTH]; + int Nline = snprintf (line, MAX_PATH_LENGTH, "md5sum %s\n", filename); + assert (Nline < MAX_PATH_LENGTH); + + write (host->stdio[0], line, Nline); + + InitIOBuffer (&buffer, 100); + int status = EmptyIOBuffer (&buffer, 10000, host->stdio[1]); + fprintf (stderr, "md5sum status: %d\n", status); + + fprintf (stderr, "buffer: %s\n", buffer.buffer); + + char result[MAX_PATH_LENGTH], fileout[MAX_PATH_LENGTH]; + int Nscan = sscanf (buffer.buffer, "%s %s", result, fileout); + assert (Nscan == 2); + assert (strlen(result) == NDIGEST); + + memcpy (digest, result, NDIGEST); + return TRUE; +} + +// pclient version of remote client +int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest) { + + IOBuffer buffer, stdout_buf, stderr_buf; + + char line[MAX_PATH_LENGTH]; + int Nline = snprintf (line, MAX_PATH_LENGTH, "job md5sum %s\n", filename); + assert (Nline < MAX_PATH_LENGTH); + + // fprintf (stderr, "command: %s\n", line); + + InitIOBuffer (&buffer, 100); + InitIOBuffer (&stdout_buf, 100); + InitIOBuffer (&stderr_buf, 100); + + // need to reset pclient for next command... + if (!PclientCommand (host, "reset", &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + FlushIOBuffer (&buffer); + + // start the md5sum command + if (!PclientCommand (host, line, &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + // XXX for the moment, abort if anything goes wrong + + // wait for result + while (!CheckBusyJob (host, &stdout_buf, &stderr_buf)) { + FlushIOBuffer (&stdout_buf); + FlushIOBuffer (&stdout_buf); + usleep (10000); + } + if (stderr_buf.Nbuffer) { + fprintf (stderr, "error in md5sum: %s\n", stderr_buf.buffer); + goto escape; + } + + // fprintf (stderr, "result from md5sum: %s\n", stdout_buf.buffer); + + char result[MAX_PATH_LENGTH], fileout[MAX_PATH_LENGTH]; + int Nscan = sscanf (stdout_buf.buffer, "%s %s", result, fileout); + if (Nscan != 2) { + fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); + goto escape; + } + if (strlen(result) != 2*NDIGEST) { + fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); + goto escape; + } + + // result is the set of 0-f nibbles, convert to the digest + int i; + for (i = 0; i < NDIGEST; i++) { + int value1 = hexchar_to_int (result[2*i+0]); + int value2 = hexchar_to_int (result[2*i+1]); + digest[i] = (value1 << 4) + value2; + // fprintf (stderr, "%02x %c %c\n", digest[i], result[2*i], result[2*i+1]); + } + + // need to reset pclient for next command... + if (!PclientCommand (host, "reset", &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + + return TRUE; + +escape: + FreeIOBuffer (&buffer); + FreeIOBuffer (&stdout_buf); + FreeIOBuffer (&stderr_buf); + return FALSE; +} + +int hexchar_to_int (char input) { + + switch (input) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return input - '0'; + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + return input - 'a' + 10; + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + return input - 'A' + 10; + default: + abort(); + } +} Index: /trunk/Ohana/src/photdbc/src/memstr.c =================================================================== --- /trunk/Ohana/src/photdbc/src/memstr.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/memstr.c (revision 33655) @@ -0,0 +1,37 @@ +# include "dvodist.h" + +// XXX move these to libohana? + +/* memstr returns a view, not an allocated string : don't free */ +/* returns pointer to start of m2 in m1, or NULL if failure */ +char *memstr (char *m1, char *m2, int n) { + + int i, N; + + N = strlen (m2); + for (i = 0; (i < n - N + 1) && memcmp (m1, m2, N); i++, m1++); + if (memcmp (m1, m2, N)) return (NULL); + return (m1); + +} + +/* formatted write statement, with intelligent allocation */ +int write_fmt (int fd, char *format, ...) { + + int Nbyte, status; + char tmp, *line; + va_list argp; + + va_start (argp, format); + Nbyte = vsnprintf (&tmp, 0, format, argp); + va_end (argp); + + va_start (argp, format); + ALLOCATE (line, char, Nbyte + 1); + vsnprintf (line, Nbyte + 1, format, argp); + status = write (fd, line, strlen(line)); + va_end (argp); + + free (line); + return (status); +} Index: /trunk/Ohana/src/photdbc/src/photdbc.c =================================================================== --- /trunk/Ohana/src/photdbc/src/photdbc.c (revision 33654) +++ /trunk/Ohana/src/photdbc/src/photdbc.c (revision 33655) @@ -3,8 +3,4 @@ int main (int argc, char **argv) { - int i; - char *skyfile; - Catalog incatalog; - Catalog outcatalog; SkyTable *sky; SkyList *skylist; @@ -17,70 +13,12 @@ // the output catalog needs to inherit the SKY_DEPTH of the input catalog - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, TRUE, SKY_DEPTH_HST, VERBOSE); + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, 0, VERBOSE); SkyTableSetFilenames (sky, CATDIR, "cpt"); skylist = SkyListByPatch (sky, -1, ®ION); - for (i = 0; i < skylist[0].Nregions; i++) { - if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); - // set the parameters which guide catalog open/load/create - incatalog.filename = skylist[0].filename[i]; - incatalog.Nsecfilt = GetPhotcodeNsecfilt (); - incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; + // hostID is 0 for master program + photdbc_catalogs (argv[1], skylist, 0); - // an error exit status here is a significant error - if (!dvo_catalog_open (&incatalog, skylist[0].regions[i], VERBOSE, "r")) { - fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", incatalog.filename); - exit (2); - } - // skip empty input catalogs - if (!incatalog.Naves_disk) { - dvo_catalog_unlock (&incatalog); - dvo_catalog_free (&incatalog); - continue; - } - - // create output catalog filename - outcatalog.filename = strsubs (incatalog.filename, CATDIR, argv[1]); - if (outcatalog.filename == NULL) Shutdown ("error with input catalog name"); - - // define outcatalog open parameters - outcatalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data - outcatalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data - outcatalog.Nsecfilt = incatalog.Nsecfilt; // inherit from the incatalog - outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; - - // output catalogs always represent the same skyregions as the input catalogs - if (!dvo_catalog_open (&outcatalog, skylist[0].regions[i], VERBOSE, "w")) { - fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", outcatalog.filename); - exit (2); - } - - // the output catalog needs to have the same values for 'objID' and 'sorted' as the input - outcatalog.objID = incatalog.objID; - outcatalog.sorted = incatalog.sorted; - if (!incatalog.sorted) { - fprintf (stderr, "ERROR: input db must be sorted: %s\n", incatalog.filename); - exit (2); - } - - /* limit number of measures based on selections */ - make_subcatalog (&outcatalog, &incatalog, skylist[0].regions[i]); - - // XXX add other filters here: - // join_stars (&outcatalog); - // unique_measures (catalog); - // flag_measures (&db, catalog); - // get_mags (catalog); - - dvo_catalog_save (&outcatalog, VERBOSE); - dvo_catalog_unlock (&outcatalog); - dvo_catalog_free (&outcatalog); - - dvo_catalog_unlock (&incatalog); - dvo_catalog_free (&incatalog); - } - - ALLOCATE (skyfile, char, strlen(argv[1]) + strlen("/SkyTable.fits") + 1); - sprintf (skyfile, "%s/SkyTable.fits", argv[1]); + char *skyfile = SkyTableFilename (argv[1]); SkyTableSave (sky, skyfile); exit (0); Index: /trunk/Ohana/src/photdbc/src/photdbc_catalogs.c =================================================================== --- /trunk/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/photdbc_catalogs.c (revision 33655) @@ -0,0 +1,157 @@ +# include "photdbc.h" +# define DEBUG 1 + +int photdbc_catalogs (char *outroot, SkyList *skylist, int hostID) { + + int i; + Catalog incatalog; + Catalog outcatalog; + + if (PARALLEL && !hostID) { + photdbc_parallel (outroot, skylist); + return FALSE; + } + + for (i = 0; i < skylist[0].Nregions; i++) { + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // set the parameters which guide catalog open/load/create + char hostfile[MAX_PATH_LENGTH]; + snprintf (hostfile, MAX_PATH_LENGTH, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name); + incatalog.filename = hostID ? hostfile : skylist[0].filename[i]; + incatalog.Nsecfilt = GetPhotcodeNsecfilt (); + incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF; + + // an error exit status here is a significant error + if (!dvo_catalog_open (&incatalog, skylist[0].regions[i], VERBOSE, "r")) { + fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", incatalog.filename); + exit (2); + } + // skip empty input catalogs + if (!incatalog.Naves_disk) { + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + continue; + } + + // create output catalog filename + char outfile[MAX_PATH_LENGTH]; + snprintf (outfile, MAX_PATH_LENGTH, "%s/%s.cpt", outroot, skylist[0].regions[i]->name); + outcatalog.filename = outfile; + if (outcatalog.filename == NULL) Shutdown ("error with input catalog name"); + + // define outcatalog open parameters + outcatalog.catformat = CATFORMAT ? dvo_catalog_catformat (CATFORMAT) : incatalog.catformat; + outcatalog.catmode = CATMODE ? dvo_catalog_catmode (CATMODE) : incatalog.catmode; + outcatalog.Nsecfilt = incatalog.Nsecfilt; // inherit from the incatalog + outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + + // output catalogs always represent the same skyregions as the input catalogs + if (!dvo_catalog_open (&outcatalog, skylist[0].regions[i], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", outcatalog.filename); + exit (2); + } + + // the output catalog needs to have the same values for 'objID' and 'sorted' as the input + outcatalog.objID = incatalog.objID; + outcatalog.sorted = incatalog.sorted; + if (!incatalog.sorted) { + fprintf (stderr, "ERROR: input db must be sorted: %s\n", incatalog.filename); + exit (2); + } + + /* limit number of measures based on selections */ + make_subcatalog (&outcatalog, &incatalog, skylist[0].regions[i]); + + // XXX add other filters here: + // join_stars (&outcatalog); + // unique_measures (catalog); + // flag_measures (&db, catalog); + // get_mags (catalog); + + dvo_catalog_save (&outcatalog, VERBOSE); + dvo_catalog_unlock (&outcatalog); + dvo_catalog_free (&outcatalog); + + dvo_catalog_unlock (&incatalog); + dvo_catalog_free (&incatalog); + } + + return TRUE; +} + +int photdbc_parallel (char *outroot, SkyList *skylist) { + + // launch the photdbo_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, skylist->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", skylist->hosts, CATDIR); + exit (1); + } + + int i; + for (i = 0; i < table->Nhosts; i++) { + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + // ensure that the paths are absolute path names + char *tmproot = abspath (outroot, MAX_PATH_LENGTH); + + // options / arguments that can affect relastro_client -update-objects: + char command[MAX_PATH_LENGTH]; + snprintf (command, MAX_PATH_LENGTH, "photdbc_client %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D NMEAS_MIN %d -D NMEAS_MIN_FILTERED %d -D AVE_SIGMA_LIM %f -D SIGMA_MAX %f", + tmproot, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, + REGION.Rmin, REGION.Rmax, REGION.Dmin, REGION.Dmax, + NMEAS_MIN, NMEAS_MIN_FILTERED, AVE_SIGMA_LIM, SIGMA_MAX + ); + + char tmpline[MAX_PATH_LENGTH]; + if (VERBOSE) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -v", command); strcpy (command, tmpline); } + if (ExcludeByInstMag) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -instmag %f %f", command, INST_MAG_MIN, INST_MAG_MAX); strcpy (command, tmpline); } + if (ExcludeByMinSigma) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -min-sigma %f", command, SIGMA_MIN_KEEP); strcpy (command, tmpline); } + if (ExcludeByMaxMinMag) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -maxminmag %f", command, MAX_MIN_MAG); strcpy (command, tmpline); } + if (PHOTCODE_DROP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-drop %s", command, PHOTCODE_DROP_LIST); strcpy (command, tmpline); } + if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-skip %s", command, PHOTCODE_SKIP_LIST); strcpy (command, tmpline); } + if (CATFORMAT) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-format %s", command, CATFORMAT); strcpy (command, tmpline); } + if (CATMODE) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-mode %s", command, CATMODE); strcpy (command, tmpline); } + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) continue; + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running photdbc_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the photdbc_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} Index: /trunk/Ohana/src/photdbc/src/photdbc_client.c =================================================================== --- /trunk/Ohana/src/photdbc/src/photdbc_client.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/photdbc_client.c (revision 33655) @@ -0,0 +1,25 @@ +# include "photdbc.h" + +int main (int argc, char **argv) { + + initialize_client (argc, argv); + + // load the current sky table (layout of all SkyRegions) + SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE); + if (!sky) { + fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR); + exit (2); + } + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + SkyList *skylist = SkyListByPatch (sky, -1, ®ION); + if (!skylist) { + fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR); + exit (2); + } + + // hostID is 0 for master program + photdbc_catalogs (argv[1], skylist, HOST_ID); + + exit (0); +} Index: /trunk/Ohana/src/photdbc/src/rconnect.c =================================================================== --- /trunk/Ohana/src/photdbc/src/rconnect.c (revision 33655) +++ /trunk/Ohana/src/photdbc/src/rconnect.c (revision 33655) @@ -0,0 +1,150 @@ +# include "dvodist.h" +# include +# include + +/* connection can take a while, allow up to 2 sec */ +# define CONNECT_TIMEOUT 5000 + +/* connect to host, start the shell: ssh hostname pclient -> command hostname shell + stdio is an array of file descriptors (stdio[3]) +*/ + +int rconnect (char *command, char *hostname, char *shell, int *stdio) { + + int i = 0, stdin_fd[2], stdout_fd[2], stderr_fd[2], status; + int result, waitstatus; + pid_t pid; + char *p; + char **argv; + IOBuffer buffer; + struct timespec request, remain; + + myAssert (command != NULL, "command is NULL"); + myAssert (hostname != NULL, "hostname is NULL"); + myAssert (shell != NULL, "shell is NULL"); + myAssert (stdio != NULL, "stdio is NULL"); + + bzero (stdin_fd, 2*sizeof(int)); + bzero (stdout_fd, 2*sizeof(int)); + bzero (stderr_fd, 2*sizeof(int)); + + if (pipe (stdin_fd) < 0) goto pipe_error; + if (pipe (stdout_fd) < 0) goto pipe_error; + if (pipe (stderr_fd) < 0) goto pipe_error; + + ALLOCATE (argv, char *, 4); + argv[0] = command; + argv[1] = hostname; + argv[2] = shell; + argv[3] = 0; + + pid = fork (); + if (!pid) { /* must be child process */ + if (VERBOSE) fprintf (stderr, "starting remote connection to %s...", hostname); + + /* close the other ends of the pipes */ + close (stdin_fd[1]); + close (stdout_fd[0]); + close (stderr_fd[0]); + + /* tie our ends of the pipes to stdin, stdout, stderr */ + dup2 (stdin_fd[0], STDIN_FILENO); + dup2 (stdout_fd[1], STDOUT_FILENO); + dup2 (stderr_fd[1], STDERR_FILENO); + + /* set all three unblocking */ + setvbuf (stdin, (char *) NULL, _IONBF, BUFSIZ); + setvbuf (stdout, (char *) NULL, _IONBF, BUFSIZ); + setvbuf (stderr, (char *) NULL, _IONBF, BUFSIZ); + + status = execvp (argv[0], argv); + fprintf (stderr, "error starting remote shell process\n"); + exit (60); + } + free (argv); + + /* close the other ends of the pipes */ + close (stdin_fd[0]); stdin_fd[0] = 0; + close (stdout_fd[1]); stdout_fd[1] = 0; + close (stderr_fd[1]); stderr_fd[1] = 0; + + /* make the pipes non-blocking */ + fcntl (stdin_fd[1], F_SETFL, O_NONBLOCK); + fcntl (stdout_fd[0], F_SETFL, O_NONBLOCK); + fcntl (stderr_fd[0], F_SETFL, O_NONBLOCK); + + /* perform handshake with pclient to verify alive & running */ + /** this handshake is similar to PclientCommand, but has important differences **/ + InitIOBuffer (&buffer, 0x100); + + /* send handshake command */ + status = write_fmt (stdin_fd[1], "echo CONNECTED\n"); + if ((status == -1) && (errno == EPIPE)) goto connect_error; + + /* try to get evidence connection is alive - wait upto a few seconds */ + p = NULL; + status = -1; + for (i = 0; (i < CONNECT_TIMEOUT) && (status != 0) && (p == NULL); i++) { + status = ReadtoIOBuffer (&buffer, stdout_fd[0]); + p = memstr (buffer.buffer, "CONNECTED", buffer.Nbuffer); + usleep (10000); // wait for client to be connected + } + if (status == 0) goto connect_error; + if (status == -1) goto connect_error; + if (VERBOSE) fprintf (stderr, "%d cycles to connect\n", i); + FreeIOBuffer (&buffer); + + if (VERBOSE) fprintf (stderr, "Connected\n"); + + stdio[0] = stdin_fd[1]; + stdio[1] = stdout_fd[0]; + stdio[2] = stderr_fd[0]; + + return (pid); + +pipe_error: + perror ("pipe error:"); + goto close_pipes; + +connect_error: + if (VERBOSE) fprintf (stderr, "error while connecting, status: %d, ncycles: %d\n", status, i); + + /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */ + request.tv_sec = 0; + request.tv_nsec = 100000; + + /* harvest the child process: kill & wait (< 100 ms) for exit */ + kill (pid, SIGKILL); + result = waitpid (pid, &waitstatus, WNOHANG); + for (i = 0; (i < 50) && (result == 0); i++) { + nanosleep (&request, &remain); + result = waitpid (pid, &waitstatus, WNOHANG); + } + + if ((result == -1) && (errno != ECHILD)) { + fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno); + exit (61); + } + if (result == 0) { + if (VERBOSE) fprintf (stderr, "child did not exit (rconnect)??"); + } + if (result > 0) { + if (result != pid) { + fprintf (stderr, "waitpid error: mis-matched PID (%d vs %d). programming error\n", result, pid); + exit (62); + } + if (WIFSTOPPED(waitstatus)) { + fprintf (stderr, "waitpid returns 'stopped': programming error\n"); + exit (63); + } + } + +close_pipes: + if (stdin_fd[0] != 0) close (stdin_fd[0]); + if (stdin_fd[1] != 0) close (stdin_fd[1]); + if (stdout_fd[0] != 0) close (stdout_fd[0]); + if (stdout_fd[1] != 0) close (stdout_fd[1]); + if (stderr_fd[0] != 0) close (stderr_fd[0]); + if (stderr_fd[1] != 0) close (stderr_fd[1]); + return (FALSE); +}