Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/Makefile =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/Makefile (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/Makefile (revision 33518) @@ -41,4 +41,6 @@ $(SRC)/HostTableLoad.$(ARCH).o \ $(SRC)/AssignSkyToHost.$(ARCH).o \ +$(SRC)/CopyBackupToHost.$(ARCH).o \ +$(SRC)/UseBackupForHost.$(ARCH).o \ $(SRC)/CopyToHostLocation.$(ARCH).o \ $(SRC)/CopyFromHostLocation.$(ARCH).o \ Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h (revision 33518) @@ -15,4 +15,6 @@ MODE_IN = 2, MODE_FIX = 3, + MODE_OUT_BACKUP = 4, + MODE_USE_BACKUP = 5, } ModeType; @@ -21,4 +23,7 @@ SkyRegion UserPatch; ModeType MODE; + +char *srcHostname; +char *dstHostname; void usage(); @@ -40,4 +45,7 @@ int get_md5_with_copy (char *input, char *output, md5_byte_t *digest); +int hexchar_to_int (char input); +int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest); +int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest); int CheckBusyJob (HostInfo *host, IOBuffer *stdout_buf, IOBuffer *stderr_buf); @@ -47,2 +55,5 @@ int FixSkyForHost (char *catdir, SkyList *skylist, HostTable *table); + +int UseBackupForHost (char *catdir, SkyList *skylist, HostTable *table); +int CopyBackupToHost (char *catdir, SkyList *skylist, HostTable *table); Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyBackupToHost.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyBackupToHost.c (revision 33518) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyBackupToHost.c (revision 33518) @@ -0,0 +1,184 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// copy all *.bck currently on the srcHost to dstHost +int CopyBackupToHost (char *catdir, SkyList *skylist, HostTable *table) { + + int i, j, k; + struct stat filestat; + char srcname[1024]; + char tgtname[1024]; + char outdir[1024]; + + md5_byte_t srcDigest[NDIGEST]; + md5_byte_t tgtDigest[NDIGEST]; + + uid_t uid = getuid(); + gid_t gid = getgid(); + + // get the IDs for the src and dst machines + int srcID = 0; + int srcIndex = 0; + int dstID = 0; + int dstIndex = 0; + for (i = 0; i < table->Nhosts; i++) { + if (!strcmp (table->hosts[i].hostname, srcHostname)) { srcID = table->hosts[i].hostID; srcIndex = i; } + if (!strcmp (table->hosts[i].hostname, dstHostname)) { dstID = table->hosts[i].hostID; dstIndex = i; } + } + if (!srcID) { + fprintf (stderr, "failed to find source host %s\n", srcHostname); + exit (1); + } + if (!dstID) { + fprintf (stderr, "failed to find destination host %s\n", dstHostname); + exit (1); + } + if (srcID == dstID) { + fprintf (stderr, "invalid destination host %s : matches source host %s\n", dstHostname, srcHostname); + exit (1); + } + + HostInfo *dstHost = &table->hosts[dstIndex]; + + int Nfailure = 0; + for (i = 0; i < skylist->Nregions; i++) { + // skip unassigned tables + if (skylist->regions[i]->hostID != srcID) continue; + + // skip catalogs which have already been copied + if (skylist->regions[i]->hostFlags & DATA_ON_BCK) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // assign the dstHost to the backup ID + skylist->regions[i]->backupID = dstID; + + // we are copying CATDIR/*.bck to dstHost/pathname/* + + // have to succeed on all 4 tables; if we fail on any one, mark it as failed + int success = TRUE; + + char *subdir = pathname (skylist->regions[i]->name); + sprintf (outdir, "%s/%s", dstHost->pathname, subdir); + free (subdir); + + int status = check_dir_access (outdir, DEBUG); + if (!status) { + fprintf (stderr, "failed to find / create target path\n"); + exit (2); + } + + char extname[4][16] = {"cpm", "cpt", "cps", "cpn"}; + for (j = 0; success && (j < 4); j++) { + + // set the in and out table names + sprintf (srcname, "%s/%s.%s.bck", catdir, skylist->regions[i]->name, extname[j]); + sprintf (tgtname, "%s/%s.%s", dstHost->pathname, skylist->regions[i]->name, extname[j]); + + // does srcname exist & can it be read? + + // check permission to read file + status = stat (srcname, &filestat); + if (status) { + if (errno == ENOENT) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) + perror ("stat:"); + fprintf (stderr, "failure to access file %s\n", srcname); + success = FALSE; + continue; + } + + // can we read the file? + if (uid == filestat.st_uid) { + if (filestat.st_mode & S_IRUSR) goto valid; + } + if (gid == filestat.st_gid) { + if (filestat.st_mode & S_IRGRP) goto valid; + } + if (filestat.st_mode & S_IROTH) goto valid; + + fprintf (stderr, "cannot read file %s, skipping\n", srcname); + success = FALSE; + continue; + + valid: + // read srcname, write to tgtname, get MD5 sum for srcname as we go: + // XXX files that fail here should be skipped (but don't give up) + if (!get_md5_with_copy (srcname, tgtname, srcDigest)) { + fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname); + success = FALSE; + continue; + } + + // need to re-open and re-read tgtname to check md5sum + // XXX files that fail here need to be checked + // char *absname = abspath (tgtname, 1024); + // if (!get_md5_from_pclient (host, absname, tgtDigest)) { + if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) { + fprintf (stderr, "error reading %s or getting md5\n", tgtname); + success = FALSE; + continue; + } + + // compare the two digest values + int match = TRUE; + for (k = 0; match && (k < NDIGEST); k++) { + match &= (tgtDigest[k] == srcDigest[k]); + } + + if (!match) { + if (DEBUG) { + fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname); + } + success = FALSE; + } + } + + if (!success) { + // if we failed to make a copy, remove the failure, mark the failure so we can retry + skylist->regions[i]->hostFlags |= DATA_COPY_FAILURE; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", dstHost->pathname, skylist->regions[i]->name, extname[j]); + if (!DEBUG) unlink (tgtname); + } + Nfailure ++; + } else { + skylist->regions[i]->hostFlags |= DATA_ON_BCK; + } + } + + if (!Nfailure) { + fprintf (stderr, "all tables successfully moved\n"); + } else { + fprintf (stderr, "%d tables were not moved\n", Nfailure); + } + + return (TRUE); +} + +/* state table for copy to/from location + + data on src (local catdir) : 00.hostID + data on tgt (remote catdir) : 10.hostID + failure to copy to tgt : 01.hostID + failure to copy to src : 11.hostID +*/ + +/*** + + hostID, backupID, and hostFlags: + + hostID gives the primary location of the data + backupID gives the second location of the data + + DATA_ON_TGT : 0x01 : file exists on hostID + DATA_ON_BCK : 0x02 : file exists on backupID + DATA_USE_BCK : 0x04 : read from backupID, not hostID + + DATA_COPY_FAILURE : error copying data to TGT (set for retry) + + + ***/ Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33518) @@ -4,8 +4,4 @@ # define DELETE_ORIGINAL 0 // make this a user option - -int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest); -int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest); -int hexchar_to_int (char input); // XXX collapse this and CopyFromHostLocation (only difference is name of src/tgt and bits set on success) @@ -198,132 +194,2 @@ */ -// XXX this would be better controlled if we used a remote pclient. then we could -// check things like the exit status and -int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest) { - - IOBuffer buffer; - - char line[1024]; - int Nline = snprintf (line, 1024, "md5sum %s\n", filename); - assert (Nline < 1024); - - write (host->stdio[0], line, Nline); - - InitIOBuffer (&buffer, 100); - int status = EmptyIOBuffer (&buffer, 10000, host->stdio[1]); - fprintf (stderr, "md5sum status: %d\n", status); - - fprintf (stderr, "buffer: %s\n", buffer.buffer); - - char result[1024], fileout[1024]; - int Nscan = sscanf (buffer.buffer, "%s %s", result, fileout); - assert (Nscan == 2); - assert (strlen(result) == NDIGEST); - - memcpy (digest, result, NDIGEST); - return TRUE; -} - -// pclient version of remote client -int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest) { - - IOBuffer buffer, stdout_buf, stderr_buf; - - char line[1024]; - int Nline = snprintf (line, 1024, "job md5sum %s\n", filename); - assert (Nline < 1024); - - // fprintf (stderr, "command: %s\n", line); - - InitIOBuffer (&buffer, 100); - InitIOBuffer (&stdout_buf, 100); - InitIOBuffer (&stderr_buf, 100); - - // need to reset pclient for next command... - if (!PclientCommand (host, "reset", &buffer)) goto escape; - if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; - FlushIOBuffer (&buffer); - - // start the md5sum command - if (!PclientCommand (host, line, &buffer)) goto escape; - if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; - // XXX for the moment, abort if anything goes wrong - - // wait for result - while (!CheckBusyJob (host, &stdout_buf, &stderr_buf)) { - FlushIOBuffer (&stdout_buf); - FlushIOBuffer (&stdout_buf); - usleep (10000); - } - if (stderr_buf.Nbuffer) { - fprintf (stderr, "error in md5sum: %s\n", stderr_buf.buffer); - goto escape; - } - - // fprintf (stderr, "result from md5sum: %s\n", stdout_buf.buffer); - - char result[1024], fileout[1024]; - int Nscan = sscanf (stdout_buf.buffer, "%s %s", result, fileout); - if (Nscan != 2) { - fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); - goto escape; - } - if (strlen(result) != 2*NDIGEST) { - fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); - goto escape; - } - - // result is the set of 0-f nibbles, convert to the digest - int i; - for (i = 0; i < NDIGEST; i++) { - int value1 = hexchar_to_int (result[2*i+0]); - int value2 = hexchar_to_int (result[2*i+1]); - digest[i] = (value1 << 4) + value2; - // fprintf (stderr, "%02x %c %c\n", digest[i], result[2*i], result[2*i+1]); - } - - // need to reset pclient for next command... - if (!PclientCommand (host, "reset", &buffer)) goto escape; - if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; - - return TRUE; - -escape: - FreeIOBuffer (&buffer); - FreeIOBuffer (&stdout_buf); - FreeIOBuffer (&stderr_buf); - return FALSE; -} - -int hexchar_to_int (char input) { - - switch (input) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - return input - '0'; - case 'a': - case 'b': - case 'c': - case 'd': - case 'e': - case 'f': - return input - 'a' + 10; - case 'A': - case 'B': - case 'C': - case 'D': - case 'E': - case 'F': - return input - 'A' + 10; - default: - abort(); - } -} Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/UseBackupForHost.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/UseBackupForHost.c (revision 33518) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/UseBackupForHost.c (revision 33518) @@ -0,0 +1,57 @@ +# include "dvodist.h" + +# define DEBUG 1 +# define DELETE_ORIGINAL 0 +// make this a user option + +// set DATA_USE_BCK for all tables with DATA_ON_BCK and host matches srcHost +int UseBackupForHost (char *catdir, SkyList *skylist, HostTable *table) { + + int i; + int srcID, srcIndex; + + // get the IDs for the src and dst machines + srcID = srcIndex = 0; + for (i = 0; i < table->Nhosts; i++) { + if (!strcmp (table->hosts[i].hostname, srcHostname)) { + srcID = table->hosts[i].hostID; + srcIndex = i; + } + } + if (!srcID) { + fprintf (stderr, "failed to find source host %s\n", srcHostname); + exit (1); + } + + for (i = 0; i < skylist->Nregions; i++) { + // skip unassigned tables + if (skylist->regions[i]->hostID != srcID) continue; + + // skip catalogs which have not been copied + if (!(skylist->regions[i]->hostFlags & DATA_ON_BCK)) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostFlags & DATA_COPY_FAILURE) continue; + + // set the USE_BCK flag + skylist->regions[i]->hostFlags |= DATA_USE_BCK; + } + + return (TRUE); +} + +/*** + + hostID, backupID, and hostFlags: + + hostID gives the primary location of the data + backupID gives the second location of the data + + DATA_ON_TGT : 0x01 : file exists on hostID + DATA_ON_BCK : 0x02 : file exists on backupID + DATA_USE_BCK : 0x04 : read from backupID, not hostID + + DATA_COPY_FAILURE : error copying data to TGT (set for retry) + + + ***/ Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c (revision 33518) @@ -27,24 +27,25 @@ } - if (MODE != MODE_FIX) { - CheckHostsAndPaths(hosts); - } - - // XXX bad // save once so repeated calls copy data to the same location - // XXX bad { - // XXX bad char *skyfile = SkyTableFilename (catdir); - // XXX bad SkyTableSave (sky, skyfile); - // XXX bad } + // XXX I would like to do something so a big dvodist operation is easy to recover + // XXX BUT, I cannot save the table here without some caution (byteswap to and fro) switch (MODE) { case MODE_OUT: + CheckHostsAndPaths(hosts); AssignSkyToHost (skylist, hosts); CopyToHostLocation (catdir, skylist, hosts); break; case MODE_IN: + CheckHostsAndPaths(hosts); CopyFromHostLocation (catdir, skylist, hosts); break; case MODE_FIX: FixSkyForHost (catdir, skylist, hosts); + break; + case MODE_OUT_BACKUP: + CopyBackupToHost (catdir, skylist, hosts); + break; + case MODE_USE_BACKUP: + UseBackupForHost (catdir, skylist, hosts); break; default: Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33518) @@ -20,4 +20,7 @@ fprintf (stderr, " -v : verbose mode\n"); fprintf (stderr, " -params : list the current parameters\n\n"); + + fprintf (stderr, " -out-backup (host1) to (host2) : move *.bck files for host1 to host2\n"); + fprintf (stderr, " -use-backup (host1) : use the backup file for host1\n"); exit (2); @@ -81,4 +84,29 @@ remove_argument (N, &argc, argv); } + if ((N = get_argument (argc, argv, "-out-backup"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use -fix with -in or -out options!\n"); + usage(); + } + // usage dvodist -out-backup (srcHost) to (dstHost) + MODE = MODE_OUT_BACKUP; + remove_argument (N, &argc, argv); + srcHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + remove_argument (N, &argc, argv); // remove 'to' + dstHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-use-backup"))) { + if (MODE) { + fprintf (stderr, "ERROR: cannot use -fix with -in or -out options!\n"); + usage(); + } + // usage dvodist -out-backup (srcHost) to (dstHost) + MODE = MODE_OUT_BACKUP; + remove_argument (N, &argc, argv); + srcHostname = strcreate (argv[N]); + remove_argument (N, &argc, argv); + } if (!MODE) usage(); Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c (revision 33517) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c (revision 33518) @@ -54,2 +54,132 @@ } +// XXX this would be better controlled if we used a remote pclient. then we could +// check things like the exit status and +int get_md5_from_remote (HostInfo *host, char *filename, md5_byte_t *digest) { + + IOBuffer buffer; + + char line[1024]; + int Nline = snprintf (line, 1024, "md5sum %s\n", filename); + assert (Nline < 1024); + + write (host->stdio[0], line, Nline); + + InitIOBuffer (&buffer, 100); + int status = EmptyIOBuffer (&buffer, 10000, host->stdio[1]); + fprintf (stderr, "md5sum status: %d\n", status); + + fprintf (stderr, "buffer: %s\n", buffer.buffer); + + char result[1024], fileout[1024]; + int Nscan = sscanf (buffer.buffer, "%s %s", result, fileout); + assert (Nscan == 2); + assert (strlen(result) == NDIGEST); + + memcpy (digest, result, NDIGEST); + return TRUE; +} + +// pclient version of remote client +int get_md5_from_pclient (HostInfo *host, char *filename, md5_byte_t *digest) { + + IOBuffer buffer, stdout_buf, stderr_buf; + + char line[1024]; + int Nline = snprintf (line, 1024, "job md5sum %s\n", filename); + assert (Nline < 1024); + + // fprintf (stderr, "command: %s\n", line); + + InitIOBuffer (&buffer, 100); + InitIOBuffer (&stdout_buf, 100); + InitIOBuffer (&stderr_buf, 100); + + // need to reset pclient for next command... + if (!PclientCommand (host, "reset", &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + FlushIOBuffer (&buffer); + + // start the md5sum command + if (!PclientCommand (host, line, &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + // XXX for the moment, abort if anything goes wrong + + // wait for result + while (!CheckBusyJob (host, &stdout_buf, &stderr_buf)) { + FlushIOBuffer (&stdout_buf); + FlushIOBuffer (&stdout_buf); + usleep (10000); + } + if (stderr_buf.Nbuffer) { + fprintf (stderr, "error in md5sum: %s\n", stderr_buf.buffer); + goto escape; + } + + // fprintf (stderr, "result from md5sum: %s\n", stdout_buf.buffer); + + char result[1024], fileout[1024]; + int Nscan = sscanf (stdout_buf.buffer, "%s %s", result, fileout); + if (Nscan != 2) { + fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); + goto escape; + } + if (strlen(result) != 2*NDIGEST) { + fprintf (stderr, "error reading md5sum: %s\n", stdout_buf.buffer); + goto escape; + } + + // result is the set of 0-f nibbles, convert to the digest + int i; + for (i = 0; i < NDIGEST; i++) { + int value1 = hexchar_to_int (result[2*i+0]); + int value2 = hexchar_to_int (result[2*i+1]); + digest[i] = (value1 << 4) + value2; + // fprintf (stderr, "%02x %c %c\n", digest[i], result[2*i], result[2*i+1]); + } + + // need to reset pclient for next command... + if (!PclientCommand (host, "reset", &buffer)) goto escape; + if (!PclientResponse (host, PCLIENT_PROMPT, &buffer)) goto escape; + + return TRUE; + +escape: + FreeIOBuffer (&buffer); + FreeIOBuffer (&stdout_buf); + FreeIOBuffer (&stderr_buf); + return FALSE; +} + +int hexchar_to_int (char input) { + + switch (input) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return input - '0'; + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + return input - 'a' + 10; + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + return input - 'A' + 10; + default: + abort(); + } +}