IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33218


Ignore:
Timestamp:
Feb 8, 2012, 8:16:40 AM (14 years ago)
Author:
eugene
Message:

dvodist builds and has all initial features

Location:
branches/eam_branches/ipp-20111122/Ohana/src/photdbc
Files:
1 added
12 edited

Legend:

Unmodified
Added
Removed
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/Makefile

    r17284 r33218  
    1818
    1919photdbc: $(BIN)/photdbc.$(ARCH)
    20 install: $(DESTBIN)/photdbc
     20dvodist: $(BIN)/dvodist.$(ARCH)
     21install: $(DESTBIN)/photdbc $(DESTBIN)/dvodist
    2122
    2223PHOTDBC = \
     
    2930$(SRC)/join_stars.$(ARCH).o        \
    3031$(SRC)/make_subcatalog.$(ARCH).o       
     32
     33DVODIST = \
     34$(SRC)/dvodist.$(ARCH).o                \
     35$(SRC)/initialize_dvodist.$(ARCH).o     \
     36$(SRC)/Shutdown_dvodist.$(ARCH).o       \
     37$(SRC)/SetSignals.$(ARCH).o             \
     38$(SRC)/rconnect.$(ARCH).o               \
     39$(SRC)/md5_ops.$(ARCH).o                \
     40$(SRC)/md5.$(ARCH).o                    \
     41$(SRC)/memstr.$(ARCH).o                 \
     42$(SRC)/HostTableLoad.$(ARCH).o          \
     43$(SRC)/AssignSkyToHost.$(ARCH).o        \
     44$(SRC)/CopyToHostLocation.$(ARCH).o     \
     45$(SRC)/CopyFromHostLocation.$(ARCH).o
    3146
    3247OLD = \
     
    5065$(SRC)/wcatalog.$(ARCH).o
    5166
    52 
    5367$(PHOTDBC): $(INC)/photdbc.h
    5468$(BIN)/photdbc.$(ARCH): $(PHOTDBC)
     69
     70$(DVODIST): $(INC)/dvodist.h
     71$(BIN)/dvodist.$(ARCH): $(DVODIST)
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h

    r33207 r33218  
    44# include <md5.h>
    55# include <unistd.h> // needed?
     6
     7# define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } }
    68
    79// used by the md5 stuff
     
    3941int           args (int argc, char **argv);
    4042void          initialize (int argc, char **argv);
    41 void          ConfigInit (int *argc, char **argv);
    4243
     44void          LockDatabase (char *catdir);
     45HostTable    *HostTableLoad (char *catdir, char *rootname);
     46int           AssignSkyToHost (SkyList *skylist, HostTable *table);
     47int           CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table);
     48int           CopyFromHostLocation(char *catdir, SkyList *skylist, HostTable *table);
    4349
    4450int           Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2);
    45 void          set_db (FITS_DB *in);
    4651void          lock_image_db (FITS_DB *db, char *filename);
    47 void          unlock_image_db (FITS_DB *db);
    48 void          check_permissions (char *basefile);
    4952void          TrapSignal (int sig);
    5053void          SetProtect (int mode);
    5154int           SetSignals (void);
    52 int           copy_images (char *outdir);
     55int           rconnect (char *command, char *hostname, char *shell, int *stdio);
     56
     57int           write_fmt (int fd, char *format, ...);
     58char         *memstr (char *m1, char *m2, int n);
     59int           get_md5_with_copy (char *input, char *output, md5_byte_t *digest);
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c

    r33207 r33218  
    1 # include "photdbc.h"
     1# include "dvodist.h"
    22# define DEBUG 1
    33
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyFromHostLocation.c

    r33207 r33218  
    1 # include "photdbc.h"
     1# include "dvodist.h"
    22
    33# define DEBUG 1
     4# define DELETE_ORIGINAL 0
     5// make this a user option
    46
    5 int CostFromHostLocation (SkyList *skylist, HostTable *table) {
     7// XXX collapse this and CopyToHostLocation (only difference is name of src/tgt and bits set on success)
     8int CopyFromHostLocation (char *catdir, SkyList *skylist, HostTable *table) {
    69
    7   int i;
     10  int i, j, k;
    811  struct stat filestat;
    912  char srcname[1024];
    1013  char tgtname[1024];
     14   
     15  md5_byte_t srcDigest[NDIGEST];
     16  md5_byte_t tgtDigest[NDIGEST];
    1117   
    1218  uid_t uid = getuid();
     
    2935
    3036    // find the host for the table
    31     int realID = (skylist->regions[i]->hostID & DATA_HOST_ID)
     37    int realID = (skylist->regions[i]->hostID & DATA_HOST_ID);
    3238    short index = table->index[realID];
    3339    HostInfo *host = &table->hosts[index];
     
    4046
    4147      // set the in and out table names
    42       sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
    43       sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
     48      sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
     49      sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]);
    4450
    4551      // does srcname exist & can it be read?
     
    4854      int status = stat (srcname, &filestat);
    4955      if (!status) {
    50         if (errno == ENOEND) continue;  // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)
     56        if (errno == ENOENT) continue;  // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)
    5157        perror ("stat:");
    5258        fprintf (stderr, "failure to access file %s\n", srcname);
     
    6975
    7076    valid:
    71       // XXX since we have to open and read the whole file anyway,
    72       // XXX perhaps we should just write the output file?
    73        
    7477      // read srcname, write to tgtname, get MD5 sum for srcname as we go:
    75       md5_byte_t srcDigest[NDIGEST];
    76       get_md5_with_copy (srcname, tgtname, srcDigest);
     78      if (!get_md5_with_copy (srcname, tgtname, srcDigest)) {
     79        fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname);
     80        success = FALSE;
     81        continue;
     82      }
    7783       
    7884      // need to re-open and re-read tgtname to check md5sum
    79       md5_byte_t srcDigest[NDIGEST];
    80       get_md5_with_copy (tgtname, NULL, tgtDigest);
     85      if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) {
     86        fprintf (stderr, "error reading %s or getting md5\n", tgtname);
     87        success = FALSE;
     88        continue;
     89      }
    8190
    8291      // compare the two digest values
    83       int match = true;
     92      int match = TRUE;
    8493      for (k = 0; match && (k < NDIGEST); k++) {
    8594        match &= (tgtDigest[k] == srcDigest[k]);
     
    92101        success = FALSE;
    93102      }
    94     }   
     103    }
    95104
    96105    if (!success) {
     
    104113      skylist->regions[i]->hostID |= DATA_COPY_FAILURE;
    105114      for (j = 0; success && (j < 4); j++) {
    106         sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
     115        sprintf (tgtname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]);
    107116        if (!DEBUG) unlink (tgtname);
    108117      }
    109118      Nfailure ++;
    110119    } else {
    111       skylist->regions[i]->hostID |= DATA_ON_TGT;
     120      skylist->regions[i]->hostID &= ~DATA_ON_TGT;
    112121      for (j = 0; success && (j < 4); j++) {
    113         sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
     122        sprintf (srcname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
    114123        if (!DEBUG && DELETE_ORIGINAL) unlink (srcname);
    115124      }
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c

    r33207 r33218  
    1 # include "photdbc.h"
     1# include "dvodist.h"
    22
    33# define DEBUG 1
     4# define DELETE_ORIGINAL 0
     5// make this a user option
    46
    5 int CostToHostLocation (SkyList *skylist, HostTable *table) {
     7// XXX collapse this and CopyFromHostLocation (only difference is name of src/tgt and bits set on success)
     8int CopyToHostLocation (char *catdir, SkyList *skylist, HostTable *table) {
    69
    7   int i;
     10  int i, j, k;
    811  struct stat filestat;
    912  char srcname[1024];
    1013  char tgtname[1024];
     14
     15  md5_byte_t srcDigest[NDIGEST];
     16  md5_byte_t tgtDigest[NDIGEST];
    1117   
    1218  uid_t uid = getuid();
     
    2935
    3036    // find the host for the table
    31     int realID = (skylist->regions[i]->hostID & DATA_HOST_ID)
     37    int realID = (skylist->regions[i]->hostID & DATA_HOST_ID);
    3238    short index = table->index[realID];
    3339    HostInfo *host = &table->hosts[index];
     
    4046
    4147      // set the in and out table names
    42       sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
     48      sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]);
    4349      sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]);
    4450
     
    4854      int status = stat (srcname, &filestat);
    4955      if (!status) {
    50         if (errno == ENOEND) continue;  // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)
     56        if (errno == ENOENT) continue;  // file does not exist (handle broken table with cpt + cpm but no cpn,cps?)
    5157        perror ("stat:");
    5258        fprintf (stderr, "failure to access file %s\n", srcname);
     
    6975
    7076    valid:
    71       // XXX since we have to open and read the whole file anyway,
    72       // XXX perhaps we should just write the output file?
    73        
    7477      // read srcname, write to tgtname, get MD5 sum for srcname as we go:
    75       md5_byte_t srcDigest[NDIGEST];
    76       get_md5_with_copy (srcname, tgtname, srcDigest);
     78      if (!get_md5_with_copy (srcname, tgtname, srcDigest)) {
     79        fprintf (stderr, "error reading %s, getting md5, or writing %s\n", srcname, tgtname);
     80        success = FALSE;
     81        continue;
     82      }
    7783       
    7884      // need to re-open and re-read tgtname to check md5sum
    79       md5_byte_t tgtDigest[NDIGEST];
    80       get_md5_with_copy (tgtname, NULL, tgtDigest);
     85      if (!get_md5_with_copy (tgtname, NULL, tgtDigest)) {
     86        fprintf (stderr, "error reading %s or getting md5\n", tgtname);
     87        success = FALSE;
     88        continue;
     89      }
    8190
    8291      // compare the two digest values
    83       int match = true;
     92      int match = TRUE;
    8493      for (k = 0; match && (k < NDIGEST); k++) {
    8594        match &= (tgtDigest[k] == srcDigest[k]);
     
    111120      skylist->regions[i]->hostID |= DATA_ON_TGT;
    112121      for (j = 0; success && (j < 4); j++) {
    113         sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]);
     122        sprintf (srcname, "%s/%s.%s", catdir, skylist->regions[i]->name, extname[j]);
    114123        if (!DEBUG && DELETE_ORIGINAL) unlink (srcname);
    115124      }
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/HostTableLoad.c

    r33207 r33218  
    1 # include "photdbc.h"
     1# include "dvodist.h"
    22# define DEBUG 1
    33
     
    2323}
    2424
    25 HostTable *HostTableLoad (char *catdir) {
     25HostTable *HostTableLoad (char *catdir, char *rootname) {
     26
     27  int i, Nline;
    2628
    2729  char *filename = NULL;
    2830
    29   ALLOCATE (filename, char, strlen(catdir) + strlen("/HostTable.dat") + 1);
    30   sprintf (filename, "%s/HostTable.dat", catdir);
     31  ALLOCATE (filename, char, strlen(catdir) + strlen(rootname) + 2); // one slash and one EOL
     32  sprintf (filename, "%s/%s", catdir, rootname);
    3133
    3234  FILE *f = fopen (filename, "r");
     
    4042  int NHOSTS = 16;
    4143  int Nhosts = 0;
     44  HostInfo *hosts = NULL;
    4245  ALLOCATE (hosts, HostInfo, NHOSTS);
    4346  InitHosts (hosts, Nhosts, NHOSTS);
     
    4548  int maxID = 0;
    4649
    47   int Nline;
    4850  for (Nline = 0; TRUE; Nline ++) {
    4951    int ID;
     
    5254    char line[1024];
    5355
    54     int status = scan_line_maxlen (f, line, 1024);
    55     if (status = EOF) break;
     56    // XXXX use this for safety: int status = scan_line_maxlen (f, line, 1024);
     57    int status = scan_line (f, line);
     58    if (status == EOF) break;
    5659
    5760    // find first non-whitespace char & skip commented lines
     
    6265    if (status != 3) {
    6366      fprintf (stderr, "error reading line %d of host table %s\n", Nline, filename);
    64       FreeHost (hosts, Nhosts);
     67      FreeHosts (hosts, Nhosts);
    6568      return NULL;
    6669    }
     
    6972
    7073    if (ID < 1) {
    71       fprint (stderr, "invalid host ID %d\n", ID);
     74      fprintf (stderr, "invalid host ID %d\n", ID);
    7275      exit (1);
    7376    }
    7477    if (ID > 65535) {
    75       fprint (stderr, "invalid host ID %d\n", ID);
     78      fprintf (stderr, "invalid host ID %d\n", ID);
    7679      exit (1);
    7780    }
     
    9093  }   
    9194
     95  HostTable *table = NULL;
    9296  ALLOCATE (table, HostTable, 1);
    9397  table->Nhosts = Nhosts;
     
    123127    if (DEBUG) fprintf (stderr, "starting host within thread\n");
    124128
    125     pid = rconnect (command, table->hosts[i].hostname, shell, stdio);
     129    int pid = rconnect (command, table->hosts[i].hostname, shell, stdio);
    126130    if (!pid) {     
    127131      /** failure to start: extend retry period **/
     
    137141    close(stdio[2]);
    138142
    139     status = check_dir_access (table->hosts[i].pathname, DEBUG);
     143    int status = check_dir_access (table->hosts[i].pathname, DEBUG);
    140144    if (!status) {
    141145      fprintf (stderr, "failed to find / create target path\n");
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/Shutdown_dvodist.c

    r33207 r33218  
    1111
    1212  // lock the image db table
    13   status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD);
     13  int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD);
    1414  if (!status) {
    1515    Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
     
    4242  va_end (argp);
    4343
    44   // XXX this is in SetSignals.c, which includes photdbc.c. 
     44  // XXX this is in SetSignals.c, which includes photdbc.h
    4545  SetProtect (TRUE);
    46   gfits_db_close (db);
    47   fprintf (stderr, "ERROR: addstar halted\n");
     46  gfits_db_close (&db);
     47  fprintf (stderr, "ERROR: dvodist halted\n");
    4848  exit (1);
    4949}
    50 
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c

    r33207 r33218  
    11# include "dvodist.h"
     2
     3// dvodist (-out | -in) (catdir)
     4// dvodist -out : copy catalog tables to distributed locations in host table (from catdir)
     5// dvodist -in  : copy catalog tables to catdir (from distributed locations in host table)
    26
    37int main (int argc, char **argv) {
    48
    5   int i;
    69  SkyTable *sky;
    710  SkyList *skylist;
    811
    912  /* get configuration info, args, lockfile */
     13  SetSignals ();
    1014  initialize (argc, argv);
    1115  char *catdir = strcreate (argv[1]);
    1216
    1317  LockDatabase (catdir);
    14 
    15   // dvodist (-out | -in)
    16   // dvodist -out : host table represents new target locations
    1718 
    18   HostTable *hosts = HostTableLoad (catdir);
    19   if (NULL) {
    20     Shutdown ("failed to load Host Table for %s\n", catdir);
    21   }
    22 
    2319  // load the current SkyTable. If SkyTable does not exist, we must fail
    2420  sky = SkyTableLoadOptimal (catdir, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE);
     
    2622  skylist = SkyListByPatch (sky, -1, &UserPatch);
    2723
     24  HostTable *hosts = HostTableLoad (catdir, sky->hosts);
     25  if (!hosts) {
     26    Shutdown ("failed to load Host Table %s for %s\n", catdir, sky->hosts);
     27  }
     28
    2829  switch (MODE) {
    2930    case MODE_OUT:
    3031      AssignSkyToHost (skylist, hosts);
    31       CopyToHostLocation (skylist, hosts);
     32      CopyToHostLocation (catdir, skylist, hosts);
    3233      break;
    3334    case MODE_IN:
    34       CopyFromHostLocation (skylist, hosts);
     35      CopyFromHostLocation (catdir, skylist, hosts);
    3536      break;
    3637    default:
     
    3940  }
    4041
    41   // XXX probably need to make this a skyregion_io.c function
    42   // char *SkyTableFilename (char *catdir);
    43   char *skyfile;
    44   ALLOCATE (skyfile, char, strlen(catdir) + strlen("/SkyTable.fits") + 1);
    45   sprintf (skyfile, "%s/SkyTable.fits", catdir);
    46   SkyTableSave (sky, filename);
     42  char *skyfile = SkyTableFilename (catdir);
     43  SkyTableSave (sky, skyfile);
    4744
    4845  exit (0);
    4946}
    50 
    51 /* things still missing AFAIK :
    52 
    53  * switch based on MODE
    54  * add header keyword to SkyTable to note existence of hostID
    55  * code for CopyFromHostLocation
    56  
    57  */
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c

    r33207 r33218  
    99  if (get_argument (argc, argv, "--help")) usage();
    1010
    11   // XXX anything? ConfigInit (&argc, argv);
    1211  args (argc, argv);
    1312}
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c

    r33207 r33218  
    1616  int fdOutput = 0;
    1717  if (output) {
    18     fOutput = fopen (tgtname, "w");
     18    fOutput = fopen (output, "w");
    1919    fdOutput = fileno (fOutput);
    2020  }
     
    2828    md5_append (&state, buffer, nbytes);
    2929    if (output) {
    30       nbytes = write (fdOutput, buffer, NBUFFER);
     30      int nbytesOut = write (fdOutput, buffer, NBUFFER);
    3131      // XXX check that we actually write out all nbytes...
     32      if (nbytesOut != nbytes) {
     33        return FALSE;
     34      }
    3235    }
    3336  }
    34   md5_finish(&state, digest);
     37  md5_finish (&state, digest);
    3538       
    3639  fclose (fInput);
    37   flush (fInput);
     40  fflush (fInput);
    3841
    3942  if (output) {
    4043    fclose (fOutput);
    41     flush (fOutput);
     44    fflush (fOutput);
    4245  }
    4346  return TRUE;
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/photdbc.c

    r31635 r33218  
    8181  }
    8282
    83   ALLOCATE (skyfile, char, strlen(argv[1]) + strlen("/SkyTable.fits") + 1);
    84   sprintf (skyfile, "%s/SkyTable.fits", argv[1]);
     83  skyfile = SkyTableFilename (argv[1]);
    8584  SkyTableSave (sky, skyfile);
    8685  exit (0);
  • branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/rconnect.c

    r33204 r33218  
    1 # include "pcontrol.h"
     1# include "dvodist.h"
     2# include <sys/types.h>
     3# include <sys/wait.h>
    24
    35/* connection can take a while, allow up to 2 sec */
     
    1820  struct timespec request, remain;
    1921
    20   ASSERT (command != NULL, "command is NULL");
    21   ASSERT (hostname != NULL, "hostname is NULL");
    22   ASSERT (shell != NULL, "shell is NULL");
    23   ASSERT (stdio != NULL, "stdio is NULL");
     22  myAssert (command != NULL, "command is NULL");
     23  myAssert (hostname != NULL, "hostname is NULL");
     24  myAssert (shell != NULL, "shell is NULL");
     25  myAssert (stdio != NULL, "stdio is NULL");
    2426
    2527  bzero (stdin_fd,  2*sizeof(int));
     
    3941  pid = fork ();
    4042  if (!pid) { /* must be child process */
    41     if (VerboseMode()) gprint (GP_ERR, "starting remote connection to %s...", hostname);
     43    if (VERBOSE) fprintf (stderr, "starting remote connection to %s...", hostname);
    4244
    4345    /* close the other ends of the pipes */
     
    5759
    5860    status = execvp (argv[0], argv);
    59     gprint (GP_ERR, "error starting remote shell process\n");
    60     pcontrol_exit (60);
     61    fprintf (stderr, "error starting remote shell process\n");
     62    exit (60);
    6163  }
    6264  free (argv);
     
    9092  if (status == 0) goto connect_error;
    9193  if (status == -1) goto connect_error;
    92   if (VerboseMode()) gprint (GP_ERR, "%d cycles to connect\n", i);
     94  if (VERBOSE) fprintf (stderr, "%d cycles to connect\n", i);
    9395  FreeIOBuffer (&buffer);
    9496
    95   if (VerboseMode()) gprint (GP_ERR, "Connected\n");
     97  if (VERBOSE) fprintf (stderr, "Connected\n");
    9698
    9799  stdio[0] = stdin_fd[1];
     
    106108
    107109connect_error:
    108   if (VerboseMode()) gprint (GP_ERR, "error while connecting, status: %d, ncycles: %d\n", status, i);
     110  if (VERBOSE) fprintf (stderr, "error while connecting, status: %d, ncycles: %d\n", status, i);
    109111
    110112  /* avoid blocking on waitpid, test every 100 usec, up to 50 msec */
     
    121123
    122124  if ((result == -1) && (errno != ECHILD)) {
    123     gprint (GP_ERR, "unexpected error from waitpid (%d): programming error\n", errno);
    124     pcontrol_exit (61);
     125    fprintf (stderr, "unexpected error from waitpid (%d): programming error\n", errno);
     126    exit (61);
    125127  }
    126128  if (result == 0) {
    127     if (VerboseMode()) gprint (GP_ERR, "child did not exit (rconnect)??");
     129    if (VERBOSE) fprintf (stderr, "child did not exit (rconnect)??");
    128130  }
    129131  if (result > 0) {
    130132    if (result != pid) {
    131       gprint (GP_ERR, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
    132       pcontrol_exit (62);
     133      fprintf (stderr, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
     134      exit (62);
    133135    }
    134136    if (WIFSTOPPED(waitstatus)) {
    135       gprint (GP_ERR, "waitpid returns 'stopped': programming error\n");
    136       pcontrol_exit (63);
     137      fprintf (stderr, "waitpid returns 'stopped': programming error\n");
     138      exit (63);
    137139    }
    138140  }
Note: See TracChangeset for help on using the changeset viewer.