Changeset 33218
- Timestamp:
- Feb 8, 2012, 8:16:40 AM (14 years ago)
- Location:
- branches/eam_branches/ipp-20111122/Ohana/src/photdbc
- Files:
-
- 1 added
- 12 edited
-
Makefile (modified) (3 diffs)
-
include/dvodist.h (modified) (2 diffs)
-
src/AssignSkyToHost.c (modified) (1 diff)
-
src/CopyFromHostLocation.c (modified) (7 diffs)
-
src/CopyToHostLocation.c (modified) (6 diffs)
-
src/HostTableLoad.c (modified) (10 diffs)
-
src/Shutdown_dvodist.c (modified) (2 diffs)
-
src/dvodist.c (modified) (3 diffs)
-
src/initialize_dvodist.c (modified) (1 diff)
-
src/md5_ops.c (modified) (2 diffs)
-
src/memstr.c (added)
-
src/photdbc.c (modified) (1 diff)
-
src/rconnect.c (modified) (7 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/Makefile
r17284 r33218 18 18 19 19 photdbc: $(BIN)/photdbc.$(ARCH) 20 install: $(DESTBIN)/photdbc 20 dvodist: $(BIN)/dvodist.$(ARCH) 21 install: $(DESTBIN)/photdbc $(DESTBIN)/dvodist 21 22 22 23 PHOTDBC = \ … … 29 30 $(SRC)/join_stars.$(ARCH).o \ 30 31 $(SRC)/make_subcatalog.$(ARCH).o 32 33 DVODIST = \ 34 $(SRC)/dvodist.$(ARCH).o \ 35 $(SRC)/initialize_dvodist.$(ARCH).o \ 36 $(SRC)/Shutdown_dvodist.$(ARCH).o \ 37 $(SRC)/SetSignals.$(ARCH).o \ 38 $(SRC)/rconnect.$(ARCH).o \ 39 $(SRC)/md5_ops.$(ARCH).o \ 40 $(SRC)/md5.$(ARCH).o \ 41 $(SRC)/memstr.$(ARCH).o \ 42 $(SRC)/HostTableLoad.$(ARCH).o \ 43 $(SRC)/AssignSkyToHost.$(ARCH).o \ 44 $(SRC)/CopyToHostLocation.$(ARCH).o \ 45 $(SRC)/CopyFromHostLocation.$(ARCH).o 31 46 32 47 OLD = \ … … 50 65 $(SRC)/wcatalog.$(ARCH).o 51 66 52 53 67 $(PHOTDBC): $(INC)/photdbc.h 54 68 $(BIN)/photdbc.$(ARCH): $(PHOTDBC) 69 70 $(DVODIST): $(INC)/dvodist.h 71 $(BIN)/dvodist.$(ARCH): $(DVODIST) -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h
r33207 r33218 4 4 # include <md5.h> 5 5 # include <unistd.h> // needed? 6 7 # define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } } 6 8 7 9 // used by the md5 stuff … … 39 41 int args (int argc, char **argv); 40 42 void initialize (int argc, char **argv); 41 void ConfigInit (int *argc, char **argv);42 43 44 void LockDatabase (char *catdir); 45 HostTable *HostTableLoad (char *catdir, char *rootname); 46 int AssignSkyToHost (SkyList *skylist, HostTable *table); 47 int CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table); 48 int CopyFromHostLocation(char *catdir, SkyList *skylist, HostTable *table); 43 49 44 50 int Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2); 45 void set_db (FITS_DB *in);46 51 void lock_image_db (FITS_DB *db, char *filename); 47 void unlock_image_db (FITS_DB *db);48 void check_permissions (char *basefile);49 52 void TrapSignal (int sig); 50 53 void SetProtect (int mode); 51 54 int SetSignals (void); 52 int copy_images (char *outdir); 55 int rconnect (char *command, char *hostname, char *shell, int *stdio); 56 57 int write_fmt (int fd, char *format, ...); 58 char *memstr (char *m1, char *m2, int n); 59 int get_md5_with_copy (char *input, char *output, md5_byte_t *digest); -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c
r33207 r33218 1 # include " photdbc.h"1 # include "dvodist.h" 2 2 # define DEBUG 1 3 3 -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyFromHostLocation.c
r33207 r33218 1 # include " photdbc.h"1 # include "dvodist.h" 2 2 3 3 # define DEBUG 1 4 # define DELETE_ORIGINAL 0 5 // make this a user option 4 6 5 int CostFromHostLocation (SkyList *skylist, HostTable *table) { 7 // XXX collapse this and CopyToHostLocation (only difference is name of src/tgt and bits set on success) 8 int CopyFromHostLocation (char *catdir, SkyList *skylist, HostTable *table) { 6 9 7 int i ;10 int i, j, k; 8 11 struct stat filestat; 9 12 char srcname[1024]; 10 13 char tgtname[1024]; 14 15 md5_byte_t srcDigest[NDIGEST]; 16 md5_byte_t tgtDigest[NDIGEST]; 11 17 12 18 uid_t uid = getuid(); … … 29 35 30 36 // find the host for the table 31 int realID = (skylist->regions[i]->hostID & DATA_HOST_ID) 37 int realID = (skylist->regions[i]->hostID & DATA_HOST_ID); 32 38 short index = table->index[realID]; 33 39 HostInfo *host = &table->hosts[index]; … … 40 46 41 47 // set the in and out table names 42 sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);43 sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);48 sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); 49 sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); 44 50 45 51 // does srcname exist & can it be read? … … 48 54 int status = stat (srcname, &filestat); 49 55 if (!status) { 50 if (errno == ENOEN D) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)56 if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) 51 57 perror ("stat:"); 52 58 fprintf (stderr, "failure to access file %s\n", srcname); … … 69 75 70 76 valid: 71 // XXX since we have to open and read the whole file anyway,72 // XXX perhaps we should just write the output file?73 74 77 // read srcname, write to tgtname, get MD5 sum for srcname as we go: 75 md5_byte_t srcDigest[NDIGEST]; 76 get_md5_with_copy (srcname, tgtname, srcDigest); 78 if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { 79 fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); 80 success = FALSE; 81 continue; 82 } 77 83 78 84 // need to re-open and re-read tgtname to check md5sum 79 md5_byte_t srcDigest[NDIGEST]; 80 get_md5_with_copy (tgtname, NULL, tgtDigest); 85 if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { 86 fprintf (stderr, "error reading %s or getting md5\n", tgtname); 87 success = FALSE; 88 continue; 89 } 81 90 82 91 // compare the two digest values 83 int match = true;92 int match = TRUE; 84 93 for (k = 0; match && (k < NDIGEST); k++) { 85 94 match &= (tgtDigest[k] == srcDigest[k]); … … 92 101 success = FALSE; 93 102 } 94 } 103 } 95 104 96 105 if (!success) { … … 104 113 skylist->regions[i]->hostID |= DATA_COPY_FAILURE; 105 114 for (j = 0; success && (j < 4); j++) { 106 sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);115 sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); 107 116 if (!DEBUG) unlink (tgtname); 108 117 } 109 118 Nfailure ++; 110 119 } else { 111 skylist->regions[i]->hostID |=DATA_ON_TGT;120 skylist->regions[i]->hostID &= ~DATA_ON_TGT; 112 121 for (j = 0; success && (j < 4); j++) { 113 sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);122 sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); 114 123 if (!DEBUG && DELETE_ORIGINAL) unlink (srcname); 115 124 } -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c
r33207 r33218 1 # include " photdbc.h"1 # include "dvodist.h" 2 2 3 3 # define DEBUG 1 4 # define DELETE_ORIGINAL 0 5 // make this a user option 4 6 5 int CostToHostLocation (SkyList *skylist, HostTable *table) { 7 // XXX collapse this and CopyFromHostLocation (only difference is name of src/tgt and bits set on success) 8 int CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table) { 6 9 7 int i ;10 int i, j, k; 8 11 struct stat filestat; 9 12 char srcname[1024]; 10 13 char tgtname[1024]; 14 15 md5_byte_t srcDigest[NDIGEST]; 16 md5_byte_t tgtDigest[NDIGEST]; 11 17 12 18 uid_t uid = getuid(); … … 29 35 30 36 // find the host for the table 31 int realID = (skylist->regions[i]->hostID & DATA_HOST_ID) 37 int realID = (skylist->regions[i]->hostID & DATA_HOST_ID); 32 38 short index = table->index[realID]; 33 39 HostInfo *host = &table->hosts[index]; … … 40 46 41 47 // set the in and out table names 42 sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);48 sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); 43 49 sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); 44 50 … … 48 54 int status = stat (srcname, &filestat); 49 55 if (!status) { 50 if (errno == ENOEN D) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)56 if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) 51 57 perror ("stat:"); 52 58 fprintf (stderr, "failure to access file %s\n", srcname); … … 69 75 70 76 valid: 71 // XXX since we have to open and read the whole file anyway,72 // XXX perhaps we should just write the output file?73 74 77 // read srcname, write to tgtname, get MD5 sum for srcname as we go: 75 md5_byte_t srcDigest[NDIGEST]; 76 get_md5_with_copy (srcname, tgtname, srcDigest); 78 if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { 79 fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); 80 success = FALSE; 81 continue; 82 } 77 83 78 84 // need to re-open and re-read tgtname to check md5sum 79 md5_byte_t tgtDigest[NDIGEST]; 80 get_md5_with_copy (tgtname, NULL, tgtDigest); 85 if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { 86 fprintf (stderr, "error reading %s or getting md5\n", tgtname); 87 success = FALSE; 88 continue; 89 } 81 90 82 91 // compare the two digest values 83 int match = true;92 int match = TRUE; 84 93 for (k = 0; match && (k < NDIGEST); k++) { 85 94 match &= (tgtDigest[k] == srcDigest[k]); … … 111 120 skylist->regions[i]->hostID |= DATA_ON_TGT; 112 121 for (j = 0; success && (j < 4); j++) { 113 sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);122 sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]); 114 123 if (!DEBUG && DELETE_ORIGINAL) unlink (srcname); 115 124 } -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/HostTableLoad.c
r33207 r33218 1 # include " photdbc.h"1 # include "dvodist.h" 2 2 # define DEBUG 1 3 3 … … 23 23 } 24 24 25 HostTable *HostTableLoad (char *catdir) { 25 HostTable *HostTableLoad (char *catdir, char *rootname) { 26 27 int i, Nline; 26 28 27 29 char *filename = NULL; 28 30 29 ALLOCATE (filename, char, strlen(catdir) + strlen( "/HostTable.dat") + 1);30 sprintf (filename, "%s/ HostTable.dat", catdir);31 ALLOCATE (filename, char, strlen(catdir) + strlen(rootname) + 2); // one slash and one EOL 32 sprintf (filename, "%s/%s", catdir, rootname); 31 33 32 34 FILE *f = fopen (filename, "r"); … … 40 42 int NHOSTS = 16; 41 43 int Nhosts = 0; 44 HostInfo *hosts = NULL; 42 45 ALLOCATE (hosts, HostInfo, NHOSTS); 43 46 InitHosts (hosts, Nhosts, NHOSTS); … … 45 48 int maxID = 0; 46 49 47 int Nline;48 50 for (Nline = 0; TRUE; Nline ++) { 49 51 int ID; … … 52 54 char line[1024]; 53 55 54 int status = scan_line_maxlen (f, line, 1024); 55 if (status = EOF) break; 56 // XXXX use this for safety: int status = scan_line_maxlen (f, line, 1024); 57 int status = scan_line (f, line); 58 if (status == EOF) break; 56 59 57 60 // find first non-whitespace char & skip commented lines … … 62 65 if (status != 3) { 63 66 fprintf (stderr, "error reading line %d of host table %s\n", Nline, filename); 64 FreeHost (hosts, Nhosts);67 FreeHosts (hosts, Nhosts); 65 68 return NULL; 66 69 } … … 69 72 70 73 if (ID < 1) { 71 fprint (stderr, "invalid host ID %d\n", ID);74 fprintf (stderr, "invalid host ID %d\n", ID); 72 75 exit (1); 73 76 } 74 77 if (ID > 65535) { 75 fprint (stderr, "invalid host ID %d\n", ID);78 fprintf (stderr, "invalid host ID %d\n", ID); 76 79 exit (1); 77 80 } … … 90 93 } 91 94 95 HostTable *table = NULL; 92 96 ALLOCATE (table, HostTable, 1); 93 97 table->Nhosts = Nhosts; … … 123 127 if (DEBUG) fprintf (stderr, "starting host within thread\n"); 124 128 125 pid = rconnect (command, table->hosts[i].hostname, shell, stdio);129 int pid = rconnect (command, table->hosts[i].hostname, shell, stdio); 126 130 if (!pid) { 127 131 /** failure to start: extend retry period **/ … … 137 141 close(stdio[2]); 138 142 139 status = check_dir_access (table->hosts[i].pathname, DEBUG);143 int status = check_dir_access (table->hosts[i].pathname, DEBUG); 140 144 if (!status) { 141 145 fprintf (stderr, "failed to find / create target path\n"); -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/Shutdown_dvodist.c
r33207 r33218 11 11 12 12 // lock the image db table 13 status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD);13 int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD); 14 14 if (!status) { 15 15 Shutdown ("ERROR: failure to lock image catalog %s", db.filename); … … 42 42 va_end (argp); 43 43 44 // XXX this is in SetSignals.c, which includes photdbc. c.44 // XXX this is in SetSignals.c, which includes photdbc.h 45 45 SetProtect (TRUE); 46 gfits_db_close ( db);47 fprintf (stderr, "ERROR: addstarhalted\n");46 gfits_db_close (&db); 47 fprintf (stderr, "ERROR: dvodist halted\n"); 48 48 exit (1); 49 49 } 50 -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c
r33207 r33218 1 1 # include "dvodist.h" 2 3 // dvodist (-out | -in) (catdir) 4 // dvodist -out : copy catalog tables to distributed locations in host table (from catdir) 5 // dvodist -in : copy catalog tables to catdir (from distributed locations in host table) 2 6 3 7 int main (int argc, char **argv) { 4 8 5 int i;6 9 SkyTable *sky; 7 10 SkyList *skylist; 8 11 9 12 /* get configuration info, args, lockfile */ 13 SetSignals (); 10 14 initialize (argc, argv); 11 15 char *catdir = strcreate (argv[1]); 12 16 13 17 LockDatabase (catdir); 14 15 // dvodist (-out | -in)16 // dvodist -out : host table represents new target locations17 18 18 HostTable *hosts = HostTableLoad (catdir);19 if (NULL) {20 Shutdown ("failed to load Host Table for %s\n", catdir);21 }22 23 19 // load the current SkyTable. If SkyTable does not exist, we must fail 24 20 sky = SkyTableLoadOptimal (catdir, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE); … … 26 22 skylist = SkyListByPatch (sky, -1, &UserPatch); 27 23 24 HostTable *hosts = HostTableLoad (catdir, sky->hosts); 25 if (!hosts) { 26 Shutdown ("failed to load Host Table %s for %s\n", catdir, sky->hosts); 27 } 28 28 29 switch (MODE) { 29 30 case MODE_OUT: 30 31 AssignSkyToHost (skylist, hosts); 31 CopyToHostLocation ( skylist, hosts);32 CopyToHostLocation (catdir, skylist, hosts); 32 33 break; 33 34 case MODE_IN: 34 CopyFromHostLocation ( skylist, hosts);35 CopyFromHostLocation (catdir, skylist, hosts); 35 36 break; 36 37 default: … … 39 40 } 40 41 41 // XXX probably need to make this a skyregion_io.c function 42 // char *SkyTableFilename (char *catdir); 43 char *skyfile; 44 ALLOCATE (skyfile, char, strlen(catdir) + strlen("/SkyTable.fits") + 1); 45 sprintf (skyfile, "%s/SkyTable.fits", catdir); 46 SkyTableSave (sky, filename); 42 char *skyfile = SkyTableFilename (catdir); 43 SkyTableSave (sky, skyfile); 47 44 48 45 exit (0); 49 46 } 50 51 /* things still missing AFAIK :52 53 * switch based on MODE54 * add header keyword to SkyTable to note existence of hostID55 * code for CopyFromHostLocation56 57 */ -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c
r33207 r33218 9 9 if (get_argument (argc, argv, "--help")) usage(); 10 10 11 // XXX anything? ConfigInit (&argc, argv);12 11 args (argc, argv); 13 12 } -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c
r33207 r33218 16 16 int fdOutput = 0; 17 17 if (output) { 18 fOutput = fopen ( tgtname, "w");18 fOutput = fopen (output, "w"); 19 19 fdOutput = fileno (fOutput); 20 20 } … … 28 28 md5_append (&state, buffer, nbytes); 29 29 if (output) { 30 nbytes= write (fdOutput, buffer, NBUFFER);30 int nbytesOut = write (fdOutput, buffer, NBUFFER); 31 31 // XXX check that we actually write out all nbytes... 32 if (nbytesOut != nbytes) { 33 return FALSE; 34 } 32 35 } 33 36 } 34 md5_finish (&state, digest);37 md5_finish (&state, digest); 35 38 36 39 fclose (fInput); 37 f lush (fInput);40 fflush (fInput); 38 41 39 42 if (output) { 40 43 fclose (fOutput); 41 f lush (fOutput);44 fflush (fOutput); 42 45 } 43 46 return TRUE; -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/photdbc.c
r31635 r33218 81 81 } 82 82 83 ALLOCATE (skyfile, char, strlen(argv[1]) + strlen("/SkyTable.fits") + 1); 84 sprintf (skyfile, "%s/SkyTable.fits", argv[1]); 83 skyfile = SkyTableFilename (argv[1]); 85 84 SkyTableSave (sky, skyfile); 86 85 exit (0); -
branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/rconnect.c
r33204 r33218 1 # include "pcontrol.h" 1 # include "dvodist.h" 2 # include <sys/types.h> 3 # include <sys/wait.h> 2 4 3 5 /* connection can take a while, allow up to 2 sec */ … … 18 20 struct timespec request, remain; 19 21 20 ASSERT(command != NULL, "command is NULL");21 ASSERT(hostname != NULL, "hostname is NULL");22 ASSERT(shell != NULL, "shell is NULL");23 ASSERT(stdio != NULL, "stdio is NULL");22 myAssert (command != NULL, "command is NULL"); 23 myAssert (hostname != NULL, "hostname is NULL"); 24 myAssert (shell != NULL, "shell is NULL"); 25 myAssert (stdio != NULL, "stdio is NULL"); 24 26 25 27 bzero (stdin_fd, 2*sizeof(int)); … … 39 41 pid = fork (); 40 42 if (!pid) { /* must be child process */ 41 if (V erboseMode()) gprint (GP_ERR, "starting remote connection to %s...", hostname);43 if (VERBOSE) fprintf (stderr, "starting remote connection to %s...", hostname); 42 44 43 45 /* close the other ends of the pipes */ … … 57 59 58 60 status = execvp (argv[0], argv); 59 gprint (GP_ERR, "error starting remote shell process\n");60 pcontrol_exit (60);61 fprintf (stderr, "error starting remote shell process\n"); 62 exit (60); 61 63 } 62 64 free (argv); … … 90 92 if (status == 0) goto connect_error; 91 93 if (status == -1) goto connect_error; 92 if (V erboseMode()) gprint (GP_ERR, "%d cycles to connect\n", i);94 if (VERBOSE) fprintf (stderr, "%d cycles to connect\n", i); 93 95 FreeIOBuffer (&buffer); 94 96 95 if (V erboseMode()) gprint (GP_ERR, "Connected\n");97 if (VERBOSE) fprintf (stderr, "Connected\n"); 96 98 97 99 stdio[0] = stdin_fd[1]; … … 106 108 107 109 connect_error: 108 if (V erboseMode()) gprint (GP_ERR, "error while connecting, status: %d, ncycles: %d\n", status, i);110 if (VERBOSE) fprintf (stderr, "error while connecting, status: %d, ncycles: %d\n", status, i); 109 111 110 112 /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */ … … 121 123 122 124 if ((result == -1) && (errno != ECHILD)) { 123 gprint (GP_ERR, "unexpected error from waitpid (%d): programming error\n", errno);124 pcontrol_exit (61);125 fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno); 126 exit (61); 125 127 } 126 128 if (result == 0) { 127 if (V erboseMode()) gprint (GP_ERR, "child did not exit (rconnect)??");129 if (VERBOSE) fprintf (stderr, "child did not exit (rconnect)??"); 128 130 } 129 131 if (result > 0) { 130 132 if (result != pid) { 131 gprint (GP_ERR, "waitpid error: mis-matched PID (%d vs %d). programming error\n", result, pid);132 pcontrol_exit (62);133 fprintf (stderr, "waitpid error: mis-matched PID (%d vs %d). programming error\n", result, pid); 134 exit (62); 133 135 } 134 136 if (WIFSTOPPED(waitstatus)) { 135 gprint (GP_ERR, "waitpid returns 'stopped': programming error\n");136 pcontrol_exit (63);137 fprintf (stderr, "waitpid returns 'stopped': programming error\n"); 138 exit (63); 137 139 } 138 140 }
Note:
See TracChangeset
for help on using the changeset viewer.
