Changeset 38441 for trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

Timestamp:

Jun 12, 2015, 6:18:23 PM (11 years ago)

Author:

eugene

Message:

merge changes from EAM dev branch ohana.20150429

Location:

trunk/Ohana

Files:

• . (modified) (1 prop)
• src/dvomerge/src/dvomergeUpdate_catalogs.c (modified) (6 diffs)

trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

@@ -142 +142 @@
 
     // skip empty input catalogs
-    if (!incatalog.Naves_disk) {
+    if (!incatalog.Naverage_disk) {
         dvo_catalog_unlock (&incatalog);
         dvo_catalog_free (&incatalog);
@@ -176 +176 @@
 
       // if no catalog already exists, use the input catalog to define the format or the specified format
-      if (outcatalog.Naves_disk == 0) {
-        if (UPDATE_CATFORMAT) {
-          outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
-        } else {
-          outcatalog.catformat = incatalog.catformat;
-        }
+      if (UPDATE_CATFORMAT) {
+        outcatalog.catformat = dvo_catalog_catformat (UPDATE_CATFORMAT);
+      } else {
+        outcatalog.catformat = incatalog.catformat;
+      }
+      if (UPDATE_CATCOMPRESS) {
+        outcatalog.catcompress = dvo_catalog_catcompress (UPDATE_CATCOMPRESS);
+      } else {
+        outcatalog.catcompress = incatalog.catcompress;
       }
 
@@ -227 +230 @@
   return TRUE;
 }
+
+int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput);
 
 // launch the dvomergeUpdate_client jobs to the parallel hosts
@@ -254 +259 @@
   }    
 
+  int Ngroups;
+  HostTableGroup *groups = HostTableGroupsMaxNumber (table, &Ngroups, MAX_CLIENTS);
+  // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines)
+
   int i;
-  for (i = 0; i < table->Nhosts; i++) {
+  for (i = 0; i < Ngroups; i++) {
+    // update only a max of MaxClient machines at a time
+    dvomergeUpdate_parallel_group (&groups[i], absinput, absoutput);
+  }
+  return TRUE;
+}
+
+int dvomergeUpdate_parallel_group (HostTableGroup *group, char *absinput, char *absoutput) {
+
+  int i;
+  for (i = 0; i < group->Nhosts; i++) {
 
     // ensure that the paths are absolute path names
-    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
-    free (table->hosts[i].pathname);
-    table->hosts[i].pathname = tmppath;
+    char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH);
+    free (group->hosts[i][0].pathname);
+    group->hosts[i][0].pathname = tmppath;
 
     // options / arguments that can affect relastro_client -update-objects:
     char command[DVO_MAX_PATH];
     snprintf (command, DVO_MAX_PATH, "dvomerge_client %s into %s -hostID %d -hostdir %s -region %f %f %f %f -D ADDSTAR_RADIUS %f", 
-              absinput, absoutput, table->hosts[i].hostID, table->hosts[i].pathname, 
+              absinput, absoutput, group->hosts[i][0].hostID, group->hosts[i][0].pathname, 
               UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, RADIUS
       );
@@ -291 +310 @@
       int status = system (command);
       if (status) {
-        fprintf (stderr, "ERROR running photdbc_client\n");
+        fprintf (stderr, "ERROR running dvomerge_client\n");
         exit (2);
       }
@@ -297 +316 @@
       // launch the job on the remote machine (no handshake)
       int errorInfo = 0;
-      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
+      int pid = rconnect ("ssh", group->hosts[i][0].hostname, command, group->hosts[i][0].stdio, &errorInfo, FALSE);
       if (!pid) {
-        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", group->hosts[i][0].hostname, errorInfo);
         exit (1);
       }
-      table->hosts[i].pid = pid; // save for future reference
+      group->hosts[i][0].pid = pid; // save for future reference
     }
   }
 
   if (PARALLEL_MANUAL) {
-    fprintf (stderr, "run the photdbc_client commands above.  when these are done, hit return\n");
+    fprintf (stderr, "run the dvomerge_client commands above.  when these are done, hit return\n");
     getchar();
   }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
-    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+    int status = HostTableGroupWaitJobsGetIO (group, __FILE__, __LINE__, VERBOSE);
     if (!status) {
       fprintf (stderr, "error running one of the remote clients\n");