Changeset 35333

Timestamp:

Mar 29, 2013, 2:47:47 PM (13 years ago)

Author:

eugene

Message:

adding parallel version for dvosecfilt

Location:

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge

Files:

• Makefile (modified) (4 diffs)
• include/dvomerge.h (modified) (1 diff)
• src/args.c (modified) (2 diffs)
• src/dvosecfilt.c (modified) (2 diffs)
• src/dvosecfilt_catalogs.c (added)
• src/dvosecfilt_client.c (added)
• src/help.c (modified) (2 diffs)

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge/Makefile

@@ -23 +23 @@
 dvoverify_client : $(BIN)/dvoverify_client.$(ARCH)
 
-all: dvomerge dvomerge_client dvoconvert dvosecfilt dvoverify dvoverify_client
+all: dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvoverify dvoverify_client
 
 #  $(SRC)/dvomergeContinue.$(ARCH).o
@@ -93 +93 @@
 DVOSECFILT = \
 $(SRC)/dvosecfilt.$(ARCH).o \
+$(SRC)/dvosecfilt_catalogs.$(ARCH).o \
 $(SRC)/SetSignals.$(ARCH).o \
 $(SRC)/ConfigInit.$(ARCH).o \
@@ -101 +102 @@
 $(DVOSECFILT)  : $(INC)/dvomerge.h
 $(BIN)/dvosecfilt.$(ARCH) : $(DVOSECFILT)
+
+DVOSECFILT_CLIENT = \
+$(SRC)/dvosecfilt_client.$(ARCH).o \
+$(SRC)/dvosecfilt_catalogs.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/ConfigInit.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
+$(SRC)/help.$(ARCH).o \
+$(SRC)/args.$(ARCH).o
+
+$(DVOSECFILT_CLIENT)  : $(INC)/dvomerge.h
+$(BIN)/dvosecfilt_client.$(ARCH) : $(DVOSECFILT_CLIENT)
 
 DVOREPAIR = \
@@ -136 +149 @@
 $(BIN)/dvoverify_client.$(ARCH) : $(DVOVERIFY_CLIENT)
 
-INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify dvoverify_client
+INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvosecfilt_client dvorepair dvoverify dvoverify_client
 
 # dependancy rules for binary code #########################

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge/include/dvomerge.h

@@ -104 +104 @@
 int        dvosecfilt_args        PROTO((int *argc, char **argv));
 
+void       dvosecfilt_client_usage PROTO((void));
+void       dvosecfilt_client_help  PROTO((int argc, char **argv));
+int        dvosecfilt_client_args  PROTO((int *argc, char **argv));
+
+int        dvosecfilt_catalogs     PROTO((SkyTable *insky, int Nsecfilt));
+int        dvosecfilt_parallel     PROTO((SkyTable *insky, int Nsecfilt));
+
 int        SkyTablePopulatedRange PROTO((off_t *ns, off_t *ne, SkyTable *sky, off_t Nstart));
 int        SkyListPopulatedRange  PROTO((off_t *ns, off_t *ne, SkyList *sky, off_t Nstart));

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge/src/args.c

@@ -170 +170 @@
   int N;
 
-  /* extra error messages */
-  VERBOSE = FALSE;
-  if ((N = get_argument (*argc, argv, "-v"))) {
-    VERBOSE = TRUE;
+  HOST_ID = 0;
+
+  /* extra error messages */
+  VERBOSE = FALSE;
+  if ((N = get_argument (*argc, argv, "-v"))) {
+    VERBOSE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  PARALLEL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel"))) {
+    PARALLEL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the remote jobs and waiting for completion,
+  // relphot will simply list the remote command and wait for the user to signal completion
+  PARALLEL_MANUAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
+    PARALLEL = TRUE; // -parallel-manual implies -parallel
+    PARALLEL_MANUAL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the relphot_client jobs remotely, they are 
+  // run in serial via 'system'
+  PARALLEL_SERIAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
+    if (PARALLEL_MANUAL) {
+      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
+      exit (1);
+    }
+    PARALLEL = TRUE; // -parallel-serial implies -parallel
+    PARALLEL_SERIAL = TRUE;
     remove_argument (N, argc, argv);
   }
@@ -180 +208 @@
   return TRUE;
 }
+
+/*** check for command line options ***/
+int dvosecfilt_client_args (int *argc, char **argv) {
+  
+  int N;
+
+  // by definition, the client is not parallel 
+  PARALLEL = FALSE;
+  PARALLEL_MANUAL = FALSE;
+  PARALLEL_SERIAL = FALSE;
+
+  HOST_ID = 0;
+  if ((N = get_argument (*argc, argv, "-hostID"))) {
+    remove_argument (N, argc, argv);
+    HOST_ID = atoi (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOST_ID) dvosecfilt_client_usage();
+
+  HOSTDIR = NULL;
+  if ((N = get_argument (*argc, argv, "-hostdir"))) {
+    remove_argument (N, argc, argv);
+    HOSTDIR = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOSTDIR) dvosecfilt_client_usage();
+
+  /* extra error messages */
+  VERBOSE = FALSE;
+  if ((N = get_argument (*argc, argv, "-v"))) {
+    VERBOSE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+
+  if (*argc != 3) dvosecfilt_client_usage();
+  return TRUE;
+}

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge/src/dvosecfilt.c

@@ -3 +3 @@
 // modify the number of average photcodes : change only secfilt tables in place
 int main (int argc, char **argv) {
-
-  char filename[256], *input;
-  off_t i, j, k, Nsecfilt, NsecInput, Nstart;
-  SkyTable *insky;
-  Catalog catalog;
-  SecFilt *insec, *outsec;
 
   SetSignals ();
@@ -15 +9 @@
   dvosecfilt_args (&argc, argv);
 
-  input = argv[1];
-  Nsecfilt = atoi(argv[2]);
+  strcpy (CATDIR, argv[1]);
+  int Nsecfilt = atoi(argv[2]);
 
   // load the sky table for the existing database
-  insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
-  SkyTableSetFilenames (insky, input, "cpt");
+  SkyTable *insky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
+  SkyTableSetFilenames (insky, CATDIR, "cpt");
 
-  // loop over all input catalogs
-  for (i = 0; i < insky[0].Nregions; i++) {
-    if (!insky[0].regions[i].table) continue;
-    if (VERBOSE) fprintf (stderr, "table: %s\n", insky[0].regions[i].name);
-
-    // set the parameters which guide catalog open/load/create
-    catalog.filename  = insky[0].filename[i];
-    catalog.Nsecfilt  = 0;
-
-    // always load all of the data (if any exists)
-    catalog.catflags  = LOAD_SECF;
-    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-
-    if (!dvo_catalog_open (&catalog, &insky[0].regions[i], VERBOSE, "w")) {
-        fprintf (stderr, "ERROR: failure to open catalog file %s\n", filename);
-        exit (2);
-    }
-
-    if (catalog.Naves_disk == 0) {
-      dvo_catalog_unlock (&catalog);
-      dvo_catalog_free (&catalog);
-      continue;
-    }
-
-    if (Nsecfilt == catalog.Nsecfilt) {
-      dvo_catalog_unlock (&catalog);
-      dvo_catalog_free (&catalog);
-      continue;
-    }
-
-    NsecInput = catalog.Nsecfilt;
-    Nstart = MIN(Nsecfilt, NsecInput);
-
-    insec = catalog.secfilt;
-    ALLOCATE (outsec, SecFilt, catalog.Naves_disk * Nsecfilt);
-
-    for (k = 0; k < catalog.Naves_disk; k++) {
-      for (j = 0; (j < catalog.Nsecfilt) && (j < Nsecfilt); j++) {
-        outsec[k*Nsecfilt + j].M  = insec[k*NsecInput + j].M;
-        outsec[k*Nsecfilt + j].dM = insec[k*NsecInput + j].dM;
-        outsec[k*Nsecfilt + j].Xm = insec[k*NsecInput + j].Xm;
-        outsec[k*Nsecfilt + j].flags  = insec[k*NsecInput + j].flags;
-        outsec[k*Nsecfilt + j].Ncode  = insec[k*NsecInput + j].Ncode;
-        outsec[k*Nsecfilt + j].Nused  = insec[k*NsecInput + j].Nused;
-        outsec[k*Nsecfilt + j].M_20   = insec[k*NsecInput + j].M_20;
-        outsec[k*Nsecfilt + j].M_80   = insec[k*NsecInput + j].M_80;
-      }
-      for (j = Nstart; j < Nsecfilt; j++) {
-        outsec[k*Nsecfilt + j].M  = NAN;
-        outsec[k*Nsecfilt + j].dM = NAN;
-        outsec[k*Nsecfilt + j].Xm = NAN_S_SHORT;
-      }
-    }
-    free (catalog.secfilt);
-    catalog.secfilt = outsec;
-    catalog.Nsecfilt = Nsecfilt;
-    catalog.Nsecf_mem = Nsecfilt * catalog.Naves_disk;
-    catalog.Nsecf_disk = Nsecfilt * catalog.Naves_disk;
-
-    dvo_catalog_save (&catalog, VERBOSE);
-    dvo_catalog_unlock (&catalog);
-    dvo_catalog_free (&catalog);
-  }
+  dvosecfilt_catalogs (insky, Nsecfilt);
 
   exit (0);

branches/eam_branches/ipp-20130307/Ohana/src/dvomerge/src/help.c

@@ -25 +25 @@
 void dvosecfilt_usage(void) {
 
-  fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n");
+  fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n\n");
+  fprintf (stderr, "  optional flags:\n");
+  fprintf (stderr, "  -v                          : verbose mode\n");
+
+  exit (2);
+}
+
+void dvosecfilt_client_usage(void) {
+
+  fprintf (stderr, "USAGE: dvosecfilt_client (catdir) (Nsecfilt)\n\n");
+  fprintf (stderr, "  optional flags:\n");
+  fprintf (stderr, "  -v                          : verbose mode\n");
 
   exit (2);
@@ -123 +134 @@
 }
 
+void dvosecfilt_client_help (int argc, char **argv) {
+
+  /* check for help request */
+  if (get_argument (argc, argv, "-help")) goto show_help;
+  if (get_argument (argc, argv, "-h"))    goto show_help;
+  return;
+
+show_help:
+
+  fprintf (stderr, "USAGE\n");
+  fprintf (stderr, "  dvosecfilt_client (catdir) (Nsecfilt)\n\n");
+
+  fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables (cps), NSECFILT in cpt files)\n");
+  fprintf (stderr, "  NOTE: the user must change the photcode table to reflect the change (use photcode-table -export / -import)\n");
+ 
+  fprintf (stderr, "  optional flags:\n");
+  fprintf (stderr, "  -v                          : verbose mode\n");
+  fprintf (stderr, "  -help                       : this list\n");
+  fprintf (stderr, "  -h                          : this list\n\n");
+  exit (2);
+}
+
 void dvorepair_help (int argc, char **argv) {