IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33308


Ignore:
Timestamp:
Feb 18, 2012, 8:44:13 PM (14 years ago)
Author:
eugene
Message:

some support functions for dvo_client; mextract example nearly complete

Location:
branches/eam_branches/ipp-20111122/Ohana/src/opihi
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.basic/shell.c

    r16895 r33308  
    1313  int exit_status;
    1414  int wait_status;
    15   int result, length;
     15  int result;
    1616  char **args, *shell;
    1717  struct timeval start, now;
    … …  
    3333  args[0] = shell;
    3434  args[1] = cmdflag;
    35 
    36   length = 0;
    37   for (i = 1; i < argc; i++) {
    38     length += strlen(argv[i]) + 1;
    39   }
    40  
    41   ALLOCATE (args[2], char, length);
    42   args[2][0] = 0;
    43   for (i = 1; i < argc; i++) {
    44     strcat (args[2], argv[i]);
    45     if (i < argc - 1) strcat (args[2], " ");
    46   }
     35  args[2] = paste_args (argc - 1, &argv[1]);
    4736  args[3] = NULL;
    4837

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.data/write_vectors.c

    r33131 r33308  
    3737  }
    3838
    39   /* open file for outuput */
    40   if (append) {
    41     f = fopen (argv[1], "a");
    42   } else {
    43     f = fopen (argv[1], "w");
    44   }
    45   if (f == (FILE *) NULL) {
    46     gprint (GP_ERR, "can't open file for write\n");
    47     return (FALSE);
    48   }
    49 
    5039  /* find number of output vectors */
    5140  Nvec = (argc - 2);
    5241  if (Nvec < 1) {
    5342    gprint (GP_ERR, "USAGE: write (file) vector vector ...\n");
    54     fclose (f);
    55     fflush (f);
    5643    return (FALSE);
    5744  }
    … …  
    6451      gprint (GP_ERR, "USAGE: write (file) vector vector ...\n");
    6552      free (vec);
    66       fclose (f);
    67       fflush (f);
    6853      return (FALSE);   
    6954    }
    … …  
    7661      gprint (GP_ERR, "error: vectors must all be the same size\n");
    7762      free (vec);
    78       fclose (f);
    79       fflush (f);
    8063      return (FALSE);   
    8164    }
    … …  
    8366
    8467  if (FITS) {
    85     char *tformat = NULL;
    86     Header header;
    87     Matrix matrix;
    88     Header theader;
    89     FTable ftable;
     68    int status = WriteVectorTableFITS (argv[1], FITS, vec, Nvec, append, format);
     69    free (vec);
     70    return status;
     71  }
    9072
    91     if (!append) {
    92       gfits_init_header (&header);
    93       header.extend = TRUE;
    94       gfits_create_header (&header);
    95       gfits_create_matrix (&header, &matrix);
    96     }
    97 
    98     gfits_create_table_header (&theader, "BINTABLE", FITS);
    99 
    100     ALLOCATE (tformat, char, 2*Nvec);
    101     if (format) {
    102       // the bintable format string can only define the byte-width of each field.  valid output columns are currently:
    103       // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double).
    104       // the format string is just the sequence of types, eg: LIIJEED
    105       // validate the format string
    106       char *ptr = format;
    107       for (j = 0; j < Nvec; j++) {
    108         while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
    109         if (*ptr == 0) {
    110           gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format);
    111           goto escape;
    112         }
    113         if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) {
    114           gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr);
    115           goto escape;
    116         }
    117         tformat[2*j + 0] = *ptr;
    118         tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
    119         ptr ++;
    120       }
    121       while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
    122       if (*ptr) {
    123         gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format);
    124         goto escape;
    125       }
    126     } else {
    127       for (j = 0; j < Nvec; j++) {
    128         // if the format is not defined, just use the native byte-widths
    129         tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J';
    130         tformat[2*j + 1] = 0;
    131       }
    132     }
    133 
    134     // define the columns of the table.  XXX NOTE: we cannot have duplicate names in
    135     // output table (because the data goes to the named column below).  need to enforce
    136     // this somehow
    137     for (j = 0; j < Nvec; j++) {
    138       gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0);
    139     }
    140     free (tformat);
    141 
    142     // generate the output array that carries the data
    143     gfits_create_table (&theader, &ftable);
    144 
    145     // add the vectors to the output array
    146     for (j = 0; j < Nvec; j++) {
    147       if (vec[j][0].type == OPIHI_FLT) {
    148         gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements);
    149       } else {
    150         gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements);
    151       }
    152     }
    153 
    154     if (!append) {
    155       gfits_fwrite_header  (f, &header);
    156       gfits_fwrite_matrix  (f, &matrix);
    157       gfits_free_header (&header);
    158       gfits_free_matrix (&matrix);
    159     }
    160     gfits_fwrite_Theader (f, &theader);
    161     gfits_fwrite_table  (f, &ftable);
    162 
    163     gfits_free_header (&theader);
    164     gfits_free_table (&ftable);
    165 
    166     fclose (f);
    167     free (vec);
    168     fflush (f);
    169     return (TRUE);
    170 
    171   escape:
    172     if (!append) {
    173       gfits_free_header (&header);
    174       gfits_free_matrix (&matrix);
    175     }
    176     gfits_free_header (&theader);
    177     gfits_free_table (&ftable);
    178 
    179     if (tformat) free (tformat);
    180     fclose (f);
    181     free (vec);
    182     fflush (f);
     73  /* open file for outuput */
     74  if (append) {
     75    f = fopen (argv[1], "a");
     76  } else {
     77    f = fopen (argv[1], "w");
     78  }
     79  if (f == (FILE *) NULL) {
     80    gprint (GP_ERR, "can't open file for write\n");
    18381    return (FALSE);
    18482  }

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo.c.in

    r16433 r33308  
    4545  }
    4646
     47  // dvo_client uses the following; they should be NULL for dvo
     48  HOST_ID = 0;
     49  HOSTDIR = NULL;
     50
    4751  return;
    4852}

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/dvo_client.c

    r33305 r33308  
    44void program_init (int *argc, char **argv) {
    55 
     6  int N;
     7
    68  /* load the commands used by this implementation */
    79  InitBasic ();
    … …  
    2325    set_int_variable ("MODULES:n", 1);
    2426    free (modules);
     27  }
     28
     29  // dvo_client should have 2 standard arguments: -hostID and -hostdir
     30  HOST_ID = 0;
     31  if ((N = get_argument (*argc, argv, "-hostID"))) {
     32    remove_argument (N, argc, argv);
     33    HOST_ID = atoi (argv[N]);;
     34    remove_argument (N, argc, argv);
     35  }
     36
     37  HOSTDIR = NULL;
     38  if ((N = get_argument (*argc, argv, "-hostdir"))) {
     39    remove_argument (N, argc, argv);
     40    HOSTDIR = strcreate (argv[N]);;
     41    remove_argument (N, argc, argv);
    2542  }
    2643

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/dvo/mextract.c

    r33305 r33308  
    11# include "dvoshell.h"
     2
     3# define DEBUG 1
     4# define PARALLEL_MANUAL 1
     5# define PARALLEL_SERIAL 0
     6# define MAX_PATH_LENGTH 1024
    27
    38int mextract (int argc, char **argv) {
    … …  
    3439  }
    3540
     41  int PARALLEL = FALSE;
     42  if ((N = get_argument (argc, argv, "-parallel"))) {
     43    remove_argument (N, &argc, argv);
     44    PARALLEL = TRUE;
     45  }
     46
     47  // dump results directly to fits file (esp for parallel dvo)
     48  char *ResultFile = NULL;
     49  if ((N = get_argument (argc, argv, "-result"))) {
     50    remove_argument (N, &argc, argv);
     51    ResultFile = strcreate(argv[N]);
     52    remove_argument (N, &argc, argv);
     53  }
     54
    3655  // XXX not yet thought through (where are these set?)
    3756  if (PARALLEL && !HOST_ID) {
    38     HostTableLaunchJobs (CATDIR, argc, argv);
    39     HostTableWaitJobs (table, __FILE__, __LINE__);
     57    // load the list of hosts
     58    SkyTable *sky = GetSkyTable();
     59    if (!sky) return FALSE;
     60
     61    char *CATDIR = GetCATDIR ();
     62    if (!CATDIR) return FALSE;
     63
     64    HostTable *table = HostTableLoad (CATDIR, sky->hosts);
     65    if (!table) {
     66      fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
     67      return FALSE;
     68    }   
     69
     70    // other things I need to append?
     71    char *basecmd = paste_args (argc, argv);
     72    double Rmin, Rmax, Dmin, Dmax;
     73    get_skyregion (&Rmin, &Rmax, &Dmin, &Dmax);
     74
     75    char tmp;
     76    char *command = NULL;
     77    int length = snprintf (&tmp, 0, "%s -D CATDIR %s -skyregion %f %f %f %f", basecmd, CATDIR, Rmin, Rmax, Dmin, Dmax);
     78
     79    ALLOCATE (command, char, length);
     80    snprintf (command, length, "%s -D CATDIR %s -skyregion %f %f %f %f", basecmd, CATDIR, Rmin, Rmax, Dmin, Dmax);
     81
     82    // launch this command remotely
     83    HostTableLaunchJobs (table, command);
     84    free (command);
     85
     86    if (PARALLEL_MANUAL) {
     87      fprintf (stderr, "run the relphot_client commands above.  when these are done, hit return\n");
     88      getchar();
     89    }
     90    if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
     91      HostTableWaitJobs (table, __FILE__, __LINE__);
     92    }
    4093
    4194    // load fields from file
    42     ReadVectorTables
     95    // ReadVectorTables
    4396    return TRUE;
    4497  }
    … …  
    132185  for (i = 0; (i < skylist[0].Nregions) && !interrupt; i++) {
    133186
    134     // XXX parallel case, wrong client
     187    // parallel case, wrong client
    135188    if (HOST_ID && (HOST_ID != skylist[0].regions[i]->hostID)) continue;
    136189
    137190    /* lock, load, unlock catalog */
    138     catalog.filename = skylist[0].filename[i];
     191    char hostfile[1024];
     192    snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
     193    catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i];
    139194    catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
    140195    catalog.Nsecfilt = Nsecfilt;
    … …  
    205260  }
    206261
    207   if (HOST_ID) {
    208     // write table
    209     WriteVectorTable (vec);
    210     return TRUE;
     262  if (ResultFile) {
     263    // write vectors to a table
     264    // char *resultFile?
     265    int status = WriteVectorTableFITS (ResultFile, "MEXTRACT", vec, Nreturn, FALSE, NULL);
     266    if (!status) goto escape;
    211267  }
    212268
    … …  
    222278
    223279escape:
     280  if (ResultFile) free (ResultFile);
    224281  if (vec) free (vec);
    225282  free (values);
    … …  
    349406  return (FALSE);
    350407}
     408
     409int HostTableLaunchJobs (HostTable *table, char *basecmd) {
     410
     411  int i;
     412  for (i = 0; i < table->Nhosts; i++) {
     413
     414    // ensure that the paths are absolute path names
     415    char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
     416    free (table->hosts[i].pathname);
     417    table->hosts[i].pathname = tmppath;
     418
     419    char resultFile[MAX_PATH_LENGTH];
     420    snprintf (resultFile, MAX_PATH_LENGTH, "%s/dvo.results.fits", table->hosts[i].pathname);
     421
     422    char command[1024];
     423    snprintf (command, 1024, "%s -result %s -hostID %d -hostdir %s", basecmd, resultFile, table->hosts[i].hostID, table->hosts[i].pathname);
     424
     425    fprintf (stderr, "command: %s\n", command);
     426
     427    if (PARALLEL_MANUAL) {
     428      continue;
     429    }
     430
     431    if (PARALLEL_SERIAL) {
     432      int status = system (command);
     433      if (status) {
     434        fprintf (stderr, "ERROR running relphot_client\n");
     435        exit (2);
     436      }
     437    } else {
     438      // launch the job on the remote machine (no handshake)
     439      int errorInfo = 0;
     440      int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
     441      if (!pid) {
     442        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
     443        exit (1);
     444      }
     445      table->hosts[i].pid = pid; // save for future reference
     446    }
     447  }
     448  return TRUE;
     449}
     450

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvomath.h

    r31635 r33308  
    127127int           ListVectors           PROTO((void));
    128128Vector       *SelectVector          PROTO((char *name, int mode, int verbose));
     129int           WriteVectorTableFITS  PROTO((char *filename, char *extname, Vector **vec, int Nvec, int append, char *format));
    129130
    130131/* buffer handling */

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/dvoshell.h

    r31635 r33308  
    292292} CMPstars;
    293293
     294/** some globals used particularly by DVO_CLIENT **/
     295int   HOST_ID;
     296char *HOSTDIR;
     297
    294298/*** dvo prototypes ***/
    295299int           DetermineTypeCode     PROTO((Average *average, Measure *measure, int code));
    … …  
    354358int wordhash (char *word);
    355359
     360int HostTableLaunchJobs (HostTable *table, char *basecmd);
     361
    356362#ifdef NOT_MOVED_TO_LIBDVO
    357363// dvo DB field functions

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/include/shell.h

    r32632 r33308  
    148148char         *opihi_append              PROTO((char *output, int *Noutput, char *start, char *stop));
    149149void          interpolate_slash         PROTO((char *line));
     150char         *paste_args                PROTO((int argc, char **argv));
    150151
    151152/* macro functions (mapped to commands) */

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/VectorOps.c

    r31160 r33308  
    288288}
    289289 
     290// write a set of vectors to a FITS file
     291int WriteVectorTableFITS (char *filename, char *extname, Vector **vec, int Nvec, int append, char *format) {
     292 
     293  char *tformat = NULL;
     294  Header header;
     295  Matrix matrix;
     296  Header theader;
     297  FTable ftable;
     298
     299  int j;
     300  FILE *f = NULL;
     301
     302  /* open file for outuput */
     303  if (append) {
     304    f = fopen (filename, "a");
     305  } else {
     306    f = fopen (filename, "w");
     307  }
     308  if (f == (FILE *) NULL) {
     309    gprint (GP_ERR, "can't open file for write\n");
     310    return (FALSE);
     311  }
     312
     313  if (!append) {
     314    gfits_init_header (&header);
     315    header.extend = TRUE;
     316    gfits_create_header (&header);
     317    gfits_create_matrix (&header, &matrix);
     318  }
     319
     320  gfits_create_table_header (&theader, "BINTABLE", extname);
     321
     322  ALLOCATE (tformat, char, 2*Nvec);
     323  if (format) {
     324    // the bintable format string can only define the byte-width of each field.  valid output columns are currently:
     325    // B (char), I (16 bit short), J (32 bit int), E (32 bit float), D (64 bit double).
     326    // the format string is just the sequence of types, eg: LIIJEED
     327    // validate the format string
     328    char *ptr = format;
     329    for (j = 0; j < Nvec; j++) {
     330      while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
     331      if (*ptr == 0) {
     332        gprint (GP_ERR, "error in binary table format %s (insufficient format chars)\n", format);
     333        goto escape;
     334      }
     335      if ((*ptr != 'B') && (*ptr != 'I') && (*ptr != 'J') && (*ptr != 'D') && (*ptr != 'E')) {
     336        gprint (GP_ERR, "error in binary table format %s: invalid character %c\n", format, *ptr);
     337        goto escape;
     338      }
     339      tformat[2*j + 0] = *ptr;
     340      tformat[2*j + 1] = 0; // a bit sleazy : use a 2xN string to store N 1-byte strings
     341      ptr ++;
     342    }
     343    while (*ptr && OHANA_WHITESPACE (*ptr)) ptr++;
     344    if (*ptr) {
     345      gprint (GP_ERR, "error in binary table format %s (extra characters in format)\n", format);
     346      goto escape;
     347    }
     348  } else {
     349    for (j = 0; j < Nvec; j++) {
     350      // if the format is not defined, just use the native byte-widths
     351      tformat[2*j + 0] = (vec[j][0].type == OPIHI_FLT) ? 'D' : 'J';
     352      tformat[2*j + 1] = 0;
     353    }
     354  }
     355
     356  // define the columns of the table.  XXX NOTE: we cannot have duplicate names in
     357  // output table (because the data goes to the named column below).  need to enforce
     358  // this somehow
     359  for (j = 0; j < Nvec; j++) {
     360    gfits_define_bintable_column (&theader, &tformat[2*j], vec[j][0].name, NULL, NULL, 1.0, 0.0);
     361  }
     362  free (tformat);
     363
     364  // generate the output array that carries the data
     365  gfits_create_table (&theader, &ftable);
     366
     367  // add the vectors to the output array
     368  for (j = 0; j < Nvec; j++) {
     369    if (vec[j][0].type == OPIHI_FLT) {
     370      gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "double", vec[j][0].elements.Flt, vec[j][0].Nelements);
     371    } else {
     372      gfits_set_bintable_column_reformat (&theader, &ftable, vec[j][0].name, "int", vec[j][0].elements.Int, vec[j][0].Nelements);
     373    }
     374  }
     375
     376  if (!append) {
     377    gfits_fwrite_header  (f, &header);
     378    gfits_fwrite_matrix  (f, &matrix);
     379    gfits_free_header (&header);
     380    gfits_free_matrix (&matrix);
     381  }
     382  gfits_fwrite_Theader (f, &theader);
     383  gfits_fwrite_table  (f, &ftable);
     384
     385  gfits_free_header (&theader);
     386  gfits_free_table (&ftable);
     387
     388  fclose (f);
     389  fflush (f);
     390  return (TRUE);
     391
     392 escape:
     393  if (!append) {
     394    gfits_free_header (&header);
     395    gfits_free_matrix (&matrix);
     396  }
     397  gfits_free_header (&theader);
     398  gfits_free_table (&ftable);
     399
     400  if (tformat) free (tformat);
     401  fclose (f);
     402  fflush (f);
     403  return (FALSE);
     404}

  • branches/eam_branches/ipp-20111122/Ohana/src/opihi/lib.shell/string.c

    r31667 r33308  
    301301  *out = *in;
    302302}
     303
     304// paste together argv[0] .. argv[N] into a single string
     305char *paste_args (int argc, char **argv) {
     306
     307  int i;
     308
     309  int length = 0;
     310  for (i = 0; i < argc; i++) {
     311    length += strlen(argv[i]) + 1;
     312  }
     313 
     314  char *string = NULL;
     315  ALLOCATE (string, char, length);
     316  string[0] = 0;
     317  for (i = 0; i < argc; i++) {
     318    strcat (string, argv[i]);
     319    if (i < argc - 1) strcat (string, " ");
     320  }
     321  return string;
     322}
     323
Note: See TracChangeset for help on using the changeset viewer.