Changeset 33206

Timestamp:

Feb 6, 2012, 9:13:29 PM (14 years ago)

Author:

eugene

Message:

adding open license md5 implementation; working on dvodist

Location:

branches/eam_branches/ipp-20111122/Ohana/src/photdbc

Files:

• include/md5.h (added)
• src/AssignSkyToHost.c (modified) (1 diff)
• src/CopyToHostLocation.c (modified) (1 diff)
• src/args_dvodist.c (added)
• src/dvodist.c (modified) (2 diffs)
• src/initialize.c (modified) (1 diff)
• src/md5.c (added)

branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c

@@ -16 +16 @@
     // assign or re-assign? if a region is already assigned, do we keep it?
 
-    // XXX for the moment, do not reassign tables 
-    // XXX we may want to run this program multiple times
+    // XXX some options;
+# if (0)
+    if (CLEAR_ERRORS && (skylist->regions[i]->hostID & DATA_COPY_FAILURE)) {
+      skylist->regions[i]->hostID = 0;
+    }
+# endif
+
+    // do not reassign tables; this lets us run this program multiple times for different regions
+    // tables already assigned are left where they are
     if (skylist->regions[i]->hostID) {
       continue;
     }
+
+
 
     // assign all regions to one of the Nhost hosts

branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c

@@ -1 @@
-# include <openssl/md5.h>
-# include <unistd.h>
 # include "photdbc.h"
+# include <md5.h>
+# include <unistd.h> // needed?
+
 # define DEBUG 1
+# define NBUFFER 512
+# define NDIGEST  16
+
+# define DATA_ON_TGT       0x4000
+# define DATA_COPY_FAILURE 0x8000
+# define DATA_HOST_ID      0x3fff
 
 int CostToHostLocation (SkyList *skylist, HostTable *table) {
 
   int i;
-  uid_t uid;
-  gid_t gid;
   struct stat filestat;
   char srcname[1024];
   char tgtname[1024];
     
-  uid = getuid();
-  gid = getgid();
+  uid_t uid = getuid();
+  gid_t gid = getgid();
 
+  int Nfailure = 0;
   for (i = 0; i < skylist->Nregions; i++) {
+    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
     
     // move each table from CATDIR/name to table->hosts[].pathname/name
-
-    // XXX assign or re-assign? if a region is already assigned, do we keep it?
 
     // skip unassigned tables
     if (!skylist->regions[i]->hostID) continue;
 
+    // skip catalogs which have already been copied
+    if (skylist->regions[i]->hostID & DATA_ON_TGT) continue;
+
+    // skip catalogs which have an uncleared error
+    if (skylist->regions[i]->hostID & DATA_COPY_FAILURE) continue;
+
     // find the host for the table
-    short index = table->index[skylist->regions[i]->hostID];
+    int realID = (skylist->regions[i]->hostID & DATA_HOST_ID)
+    short index = table->index[realID];
     HostInfo *host = &table->hosts[index];
 
+    // have to succeed on all 4 tables; if we fail on any one, 
+    int success = TRUE;
+
     char extname[4][16] = {"cpm", "cpt", "cps", "cpn"};
-    for (j = 0; j < 4; j++) {
+    for (j = 0; success && (j < 4); j++) {
 
-        // set the in and out table names
-        sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname);
-        sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname);
+      // set the in and out table names
+      sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
+      sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
 
-        // does srcname exist & can it be read?
+      // does srcname exist & can it be read?
 
-        // check permission to read file 
-        int status = stat (srcname, &filestat);
-        if (!status) continue;  // file does not exist (check error stats for other issues?)
+      // check permission to read file 
+      int status = stat (srcname, &filestat);
+      if (!status) {
+        if (errno == ENOEND) continue;  // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)
+        perror ("stat:");
+        fprintf (stderr, "failure to access file %s\n", srcname);
+        success = FALSE;
+        continue;
+      }
     
-        // can we read the file
-        if (uid == filestat.st_uid) {
-            if (filestat.st_mode & S_IRUSR) goto valid;
-        }
-        if (gid == filestat.st_gid) {
-            if (filestat.st_mode & S_IRGRP) goto valid;
-        }
-        if (filestat.st_mode & S_IROTH) goto valid;
+      // can we read the file?
+      if (uid == filestat.st_uid) {
+        if (filestat.st_mode & S_IRUSR) goto valid;
+      }
+      if (gid == filestat.st_gid) {
+        if (filestat.st_mode & S_IRGRP) goto valid;
+      }
+      if (filestat.st_mode & S_IROTH) goto valid;
     
-        fprintf (stderr, "cannot read file %s, skipping\n", srcname);
-        continue;
+      fprintf (stderr, "cannot read file %s, skipping\n", srcname);
+      success = FALSE;
+      continue;
 
     valid:
-        // XXX since we have to open and read the whole file anyway, 
-        // XXX perhaps we should just write the output file?
+      // XXX since we have to open and read the whole file anyway, 
+      // XXX perhaps we should just write the output file?
         
-        // read srcname, write to tgtname, get MD5 sum for srcname as we go:
+      // read srcname, write to tgtname, get MD5 sum for srcname as we go:
+      md5_byte_t srcDigest[NDIGEST];
+      get_md5_with_copy (srcname, tgtname, srcDigest);
         
-        // open the src file
-        FILE *fInput = fopen (srcname, "r");
-        int fdInput = fileno (fInput);
-        
-        // open the tgt file
-        FILE *fOutput = fopen (tgtname, "w");
-        int fdOutput = fileno (fOutput);
+      // need to re-open and re-read tgtname to check md5sum
+      get_md5_with_copy (tgtname, NULL, tgtDigest);
 
-        int n;
-        MD5_CTX c;
-        char buf[512];
-        ssize_t nbytes;
+      // compare the two digest values
+      int match = true;
+      for (k = 0; match && (k < NDIGEST); k++) {
+        match &= (tgtDigest[k] == srcDigest[k]);
+      }
 
-        MD5_Init(&c);
-        while ((nbytes = read (fd, buf, 512)) > 0) {
-            MD5_Update(&c, buf, nbytes);
-            nbytes = write (fd, buf, 512);
+      if (!match) {
+        if (DEBUG) {
+          fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname);
         }
-        MD5_Final(out, &c);
+        success = FALSE;
+      }
+    }    
 
-        for (n = 0; n < MD5_DIGEST_LENGTH; n++) {
-            fprintf (stderr, "%02x", out[n]);
-        }
-        fclose (f);
-        flush (f);
+    if (!success) {
+      // let's use the upper 2 bits of the hostID for informational purposes. this
+      // leaves us with up to 16383 hosts.
 
-    // need to re-open and re-read tgtname to check md5sum
-
-}
-    
-
-
-    // assign all regions to one of the Nhost hosts
-    int N = MAX(table->Nhosts - 1, table->Nhosts * drand48());
-    skylist->regions[i]->hostID = table->hosts[N].hostID;
-  }    
+      // data on src  = (hostID & 0x8000) >> 15
+      // copy failed  = (hostID & 0x4000) >> 14
+      // if we failed to make a copy, 
+          
+      skylist->regions[i]->hostID |= DATA_COPY_FAILURE;
+      for (j = 0; success && (j < 4); j++) {
+        sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
+        if (!DEBUG) unlink (tgtname);
+      }
+      Nfailure ++;
+    } else {
+      skylist->regions[i]->hostID |= DATA_ON_TGT;
+      for (j = 0; success && (j < 4); j++) {
+        sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
+        if (!DEBUG && DELETE_ORIGINAL) unlink (srcname);
+      }
+    }
+  }
+  
+  if (!Nfailure) {
+    fprintf (stderr, "all tables successfully moved\n");
+  } else {
+    fprintf (stderr, "%d tables were not moved\n", Nfailure);
+  }
 
   return (TRUE);
 }
+
+/* state table for copy to/from location 
+
+   data on src (local catdir)  : 00.hostID
+   data on tgt (remote catdir) : 10.hostID
+   failure to copy to tgt      : 01.hostID
+   failure to copy to src      : 11.hostID
+*/
+
+// read a file, get the md5 sum, optionally write to 'output'
+int get_md5_with_copy (char *input, char *output, md5_byte_t *digest) {
+
+  // open the src file
+  FILE *fInput = fopen (input, "r");
+  int fdInput = fileno (fInput);
+        
+  // open the tgt file
+  FILE *fOutput = NULL;
+  int fdOutput = 0;
+  if (output) {
+    fOutput = fopen (tgtname, "w");
+    fdOutput = fileno (fOutput);
+  }
+
+  int nbytes;
+  md5_state_t state;
+  md5_byte_t buffer[NBUFFER];
+
+  md5_init (&state);
+  while ((nbytes = read (fdInput, buffer, NBUFFER)) > 0) {
+    md5_append (&state, buffer, nbytes);
+    if (output) {
+      nbytes = write (fdOutput, buffer, NBUFFER);
+      // XXX check that we actually write out all nbytes...
+    }
+  }
+  md5_finish(&state, digest);
+        
+  fclose (fInput);
+  flush (fInput);
+
+  if (output) {
+    fclose (fOutput);
+    flush (fOutput);
+  }
+  return TRUE;
+}
+
+/* need to think a bit about behavior in the context of SPLIT vs MEF:
+
+ * SPLIT : all 4 tables either exist or none do
+ * MEF only cpt table exists
+
+ * read the SPLIT headers to see if MEASURE, MISSING, SECFILT fields exist?
+ */

branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c

@@ -4 +4 @@
 
   int i;
-  char *skyfile;
-  Catalog incatalog;
-  Catalog outcatalog;
   SkyTable *sky;
   SkyList *skylist;
 
   /* get configuration info, args, lockfile */
-  initialize (argc, argv);
+  initialize_dvodist (argc, argv);
 
   // dvodist (-out | -in)
@@ -25 +22 @@
   AssignSkyToHost (skylist, hosts);
 
-  CopyToHostLocation ();
+  CopyToHostLocation (skylist, hosts);
 
-  SkyTableSave (sky, skyfile);
+  char *skyfile;
+  ALLOCATE (skyfile, char, strlen(catdir) + strlen("/SkyTable.fits") + 1);
+  sprintf (skyfile, "%s/SkyTable.fits", catdir);
+  SkyTableSave (sky, filename);
 
-}
-
-void stuff () {
-
-  for (i = 0; i < skylist[0].Nregions; i++) {
-    if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name);
-
-    // set the parameters which guide catalog open/load/create
-    incatalog.filename = skylist[0].filename[i];
-    incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
-    incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
-
-    // an error exit status here is a significant error
-    if (!dvo_catalog_open (&incatalog, skylist[0].regions[i], VERBOSE, "r")) {
-      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", incatalog.filename);
-      exit (2);
-    }
-    // skip empty input catalogs
-    if (!incatalog.Naves_disk) {
-      dvo_catalog_unlock (&incatalog);
-      dvo_catalog_free (&incatalog);
-      continue;
-    }
-
-    // create output catalog filename
-    outcatalog.filename = strsubs (incatalog.filename, CATDIR, argv[1]);
-    if (outcatalog.filename == NULL) Shutdown ("error with input catalog name");
-
-    // define outcatalog open parameters
-    outcatalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    outcatalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    outcatalog.Nsecfilt  = incatalog.Nsecfilt;                 // inherit from the incatalog
-    outcatalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
-
-    // output catalogs always represent the same skyregions as the input catalogs
-    if (!dvo_catalog_open (&outcatalog, skylist[0].regions[i], VERBOSE, "w")) {
-      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", outcatalog.filename);
-      exit (2);
-    }
-
-    // the output catalog needs to have the same values for 'objID' and 'sorted' as the input
-    outcatalog.objID = incatalog.objID;
-    outcatalog.sorted = incatalog.sorted;
-    if (!incatalog.sorted) {
-      fprintf (stderr, "ERROR: input db must be sorted: %s\n", incatalog.filename);
-      exit (2);
-    }
-
-    /* limit number of measures based on selections */
-    make_subcatalog (&outcatalog, &incatalog, skylist[0].regions[i]);
-        
-    // XXX add other filters here:
-    // join_stars (&outcatalog);
-    // unique_measures (catalog);
-    // flag_measures (&db, catalog);
-    // get_mags (catalog);
-
-    dvo_catalog_save (&outcatalog, VERBOSE);
-    dvo_catalog_unlock (&outcatalog);
-    dvo_catalog_free (&outcatalog);
-
-    dvo_catalog_unlock (&incatalog);
-    dvo_catalog_free (&incatalog);
-  }
-
-  ALLOCATE (skyfile, char, strlen(argv[1]) + strlen("/SkyTable.fits") + 1);
-  sprintf (skyfile, "%s/SkyTable.fits", argv[1]);
-  SkyTableSave (sky, skyfile);
   exit (0);
 }

branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize.c

@@ -83 +83 @@
   }
 }
-