Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h (revision 33207) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/include/dvodist.h (revision 33207) @@ -0,0 +1,52 @@ +# include +# include +# include +# include +# include // needed? + +// used by the md5 stuff +# define NDIGEST 16 + +typedef enum { + MODE_NONE = 0, + MODE_OUT = 1, + MODE_IN = 2, +} ModeType; + +typedef struct { + char *hostname; + char *pathname; + int hostID; +} HostInfo; + +typedef struct { + int Nhosts; + HostInfo *hosts; + unsigned short *index; +} HostTable; + +// these should be moved to dvo.h? +# define DATA_ON_TGT 0x4000 +# define DATA_COPY_FAILURE 0x8000 +# define DATA_HOST_ID 0x3fff + +/* global variables */ +int VERBOSE; +SkyRegion UserPatch; +ModeType MODE; + +void usage(); +int args (int argc, char **argv); +void initialize (int argc, char **argv); +void ConfigInit (int *argc, char **argv); + + +int Shutdown (char *format, ...) OHANA_FORMAT(printf, 1, 2); +void set_db (FITS_DB *in); +void lock_image_db (FITS_DB *db, char *filename); +void unlock_image_db (FITS_DB *db); +void check_permissions (char *basefile); +void TrapSignal (int sig); +void SetProtect (int mode); +int SetSignals (void); +int copy_images (char *outdir); Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c (revision 33206) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/AssignSkyToHost.c (revision 33207) @@ -29,6 +29,4 @@ } - - // assign all regions to one of the Nhost hosts int N = MAX(table->Nhosts - 1, table->Nhosts * drand48()); Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyFromHostLocation.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyFromHostLocation.c (revision 33207) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyFromHostLocation.c (revision 33207) @@ -0,0 +1,142 @@ +# include "photdbc.h" + +# define DEBUG 1 + +int CostFromHostLocation (SkyList *skylist, HostTable *table) { + + int i; + struct stat filestat; + char srcname[1024]; + char tgtname[1024]; + + uid_t uid = getuid(); + gid_t gid = getgid(); + + int Nfailure = 0; + for (i = 0; i < skylist->Nregions; i++) { + if (VERBOSE) fprintf (stderr, "%s\n", skylist[0].regions[i][0].name); + + // move each table from table->hosts[].pathname/name to CATDIR/name + + // skip unassigned tables + if (!skylist->regions[i]->hostID) continue; + + // skip catalogs which have not been copied back + if (skylist->regions[i]->hostID & DATA_ON_TGT) continue; + + // skip catalogs which have an uncleared error + if (skylist->regions[i]->hostID & DATA_COPY_FAILURE) continue; + + // find the host for the table + int realID = (skylist->regions[i]->hostID & DATA_HOST_ID) + short index = table->index[realID]; + HostInfo *host = &table->hosts[index]; + + // have to succeed on all 4 tables; if we fail on any one, + int success = TRUE; + + char extname[4][16] = {"cpm", "cpt", "cps", "cpn"}; + for (j = 0; success && (j < 4); j++) { + + // set the in and out table names + sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]); + sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + + // does srcname exist & can it be read? + + // check permission to read file + int status = stat (srcname, &filestat); + if (!status) { + if (errno == ENOEND) continue; // file does not exist (handle broken table with cpt + cpm but no cpn,cps?) + perror ("stat:"); + fprintf (stderr, "failure to access file %s\n", srcname); + success = FALSE; + continue; + } + + // can we read the file? + if (uid == filestat.st_uid) { + if (filestat.st_mode & S_IRUSR) goto valid; + } + if (gid == filestat.st_gid) { + if (filestat.st_mode & S_IRGRP) goto valid; + } + if (filestat.st_mode & S_IROTH) goto valid; + + fprintf (stderr, "cannot read file %s, skipping\n", srcname); + success = FALSE; + continue; + + valid: + // XXX since we have to open and read the whole file anyway, + // XXX perhaps we should just write the output file? + + // read srcname, write to tgtname, get MD5 sum for srcname as we go: + md5_byte_t srcDigest[NDIGEST]; + get_md5_with_copy (srcname, tgtname, srcDigest); + + // need to re-open and re-read tgtname to check md5sum + md5_byte_t srcDigest[NDIGEST]; + get_md5_with_copy (tgtname, NULL, tgtDigest); + + // compare the two digest values + int match = true; + for (k = 0; match && (k < NDIGEST); k++) { + match &= (tgtDigest[k] == srcDigest[k]); + } + + if (!match) { + if (DEBUG) { + fprintf (stderr, "failed to copy %s to %s\n", srcname, tgtname); + } + success = FALSE; + } + } + + if (!success) { + // let's use the upper 2 bits of the hostID for informational purposes. this + // leaves us with up to 16383 hosts. + + // data on src = (hostID & 0x8000) >> 15 + // copy failed = (hostID & 0x4000) >> 14 + // if we failed to make a copy, + + skylist->regions[i]->hostID |= DATA_COPY_FAILURE; + for (j = 0; success && (j < 4); j++) { + sprintf (tgtname, "%s/%s.%s", host->pathname, skylist->regions[i]->name, extname[j]); + if (!DEBUG) unlink (tgtname); + } + Nfailure ++; + } else { + skylist->regions[i]->hostID |= DATA_ON_TGT; + for (j = 0; success && (j < 4); j++) { + sprintf (srcname, "%s/%s.%s", CATDIR, skylist->regions[i]->name, extname[j]); + if (!DEBUG && DELETE_ORIGINAL) unlink (srcname); + } + } + } + + if (!Nfailure) { + fprintf (stderr, "all tables successfully moved\n"); + } else { + fprintf (stderr, "%d tables were not moved\n", Nfailure); + } + + return (TRUE); +} + +/* state table for copy to/from location + + data on src (local catdir) : 00.hostID + data on tgt (remote catdir) : 10.hostID + failure to copy to tgt : 01.hostID + failure to copy to src : 11.hostID +*/ + +/* need to think a bit about behavior in the context of SPLIT vs MEF: + + * SPLIT : all 4 tables either exist or none do + * MEF only cpt table exists + + * read the SPLIT headers to see if MEASURE, MISSING, SECFILT fields exist? + */ Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33206) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/CopyToHostLocation.c (revision 33207) @@ -1,13 +1,5 @@ # include "photdbc.h" -# include -# include // needed? # define DEBUG 1 -# define NBUFFER 512 -# define NDIGEST 16 - -# define DATA_ON_TGT 0x4000 -# define DATA_COPY_FAILURE 0x8000 -# define DATA_HOST_ID 0x3fff int CostToHostLocation (SkyList *skylist, HostTable *table) { @@ -85,4 +77,5 @@ // need to re-open and re-read tgtname to check md5sum + md5_byte_t tgtDigest[NDIGEST]; get_md5_with_copy (tgtname, NULL, tgtDigest); @@ -141,43 +134,4 @@ */ -// read a file, get the md5 sum, optionally write to 'output' -int get_md5_with_copy (char *input, char *output, md5_byte_t *digest) { - - // open the src file - FILE *fInput = fopen (input, "r"); - int fdInput = fileno (fInput); - - // open the tgt file - FILE *fOutput = NULL; - int fdOutput = 0; - if (output) { - fOutput = fopen (tgtname, "w"); - fdOutput = fileno (fOutput); - } - - int nbytes; - md5_state_t state; - md5_byte_t buffer[NBUFFER]; - - md5_init (&state); - while ((nbytes = read (fdInput, buffer, NBUFFER)) > 0) { - md5_append (&state, buffer, nbytes); - if (output) { - nbytes = write (fdOutput, buffer, NBUFFER); - // XXX check that we actually write out all nbytes... - } - } - md5_finish(&state, digest); - - fclose (fInput); - flush (fInput); - - if (output) { - fclose (fOutput); - flush (fOutput); - } - return TRUE; -} - /* need to think a bit about behavior in the context of SPLIT vs MEF: Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/HostTableLoad.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/HostTableLoad.c (revision 33206) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/HostTableLoad.c (revision 33207) @@ -1,15 +1,4 @@ # include "photdbc.h" # define DEBUG 1 - -typedef struct { - char *hostname; - char *pathname; - int hostID; -} HostInfo - -typedef struct { - int Nhosts; - HostInfo *hosts; -} HostTable void InitHosts (HostInfo *hosts, int Nhosts, int NHOSTS) { @@ -34,5 +23,10 @@ } -HostTable *HostTableLoad () { +HostTable *HostTableLoad (char *catdir) { + + char *filename = NULL; + + ALLOCATE (filename, char, strlen(catdir) + strlen("/HostTable.dat") + 1); + sprintf (filename, "%s/HostTable.dat", catdir); FILE *f = fopen (filename, "r"); Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/Shutdown_dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 33207) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/Shutdown_dvodist.c (revision 33207) @@ -0,0 +1,50 @@ +# include "dvodist.h" + +static FITS_DB db; + +void LockDatabase (char *catdir) { + + char ImageCat[1024]; + + // lock image table + snprintf (ImageCat, 1024, "%s/Images.dat", catdir); + + // lock the image db table + status = dvo_image_lock (&db, ImageCat, 60.0, LCK_XCLD); + if (!status) { + Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + } + + // if the file is missing, the function above will succeed, but the db.dbstate will have + // a value of either: LCK_EMPTY or LCK_MISSING + if ((db.dbstate == LCK_EMPTY) || (db.dbstate == LCK_MISSING)) { + Shutdown ("no images?"); + } + + // XXX this program should be able to operate on a dvo database without any images (like + // 2MASS). but, we still need to lock the database when moving the files around to + // avoid collisions. So, we probably need to create an empty image table and lock it if + // there are no images at this time. +} + +/* clean up open / locked ImageCat before shutting down */ +int Shutdown (char *format, ...) { + va_list argp; + char *formatplus; + + ALLOCATE (formatplus, char, strlen(format)); + strcpy (formatplus, format); + strcat (formatplus, "\n"); + + va_start (argp, format); + vfprintf (stderr, formatplus, argp); + free (formatplus); + va_end (argp); + + // XXX this is in SetSignals.c, which includes photdbc.c. + SetProtect (TRUE); + gfits_db_close (db); + fprintf (stderr, "ERROR: addstar halted\n"); + exit (1); +} + Index: anches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/args_dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/args_dvodist.c (revision 33206) +++ (revision ) @@ -1,80 +1,0 @@ -# include "photdbc.h" - -void initialize_dvodist (int argc, char **argv) { - - /* are these set correctly? */ - if (get_argument (argc, argv, "-h")) usage_dvodist(); - if (get_argument (argc, argv, "--h")) usage_dvodist(); - if (get_argument (argc, argv, "-help")) usage_dvodist(); - if (get_argument (argc, argv, "--help")) usage_dvodist(); - - // XXX anything? ConfigInit (&argc, argv); - args (argc, argv); -} - -void usage_dvodist() { - - fprintf (stderr, "USAGE: dvodist (-out | -in) (catdir)\n\n"); - fprintf (stderr, " this program takes an existing DVO database and distributes it to the remote machine locations\n"); - - fprintf (stderr, " optional options:\n"); - fprintf (stderr, " -v : verbose mode\n"); - fprintf (stderr, " -params : list the current parameters\n\n"); - - exit (2); -} - -int args (int argc, char **argv) { - - int N; - - VERBOSE = FALSE; - if ((N = get_argument (argc, argv, "-v"))) { - VERBOSE = TRUE; - remove_argument (N, &argc, argv); - } - - /* specify portion of the sky */ - REGION.Rmin = 0; - REGION.Rmax = 360; - REGION.Dmin = -90; - REGION.Dmax = +90; - if ((N = get_argument (argc, argv, "-region"))) { - remove_argument (N, &argc, argv); - REGION.Rmin = atof (argv[N]); - remove_argument (N, &argc, argv); - REGION.Rmax = atof (argv[N]); - remove_argument (N, &argc, argv); - REGION.Dmin = atof (argv[N]); - remove_argument (N, &argc, argv); - REGION.Dmax = atof (argv[N]); - remove_argument (N, &argc, argv); - - if (REGION.Rmin == REGION.Rmax) { - fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n"); - exit (2); - } - if (REGION.Dmin == REGION.Dmax) { - fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n"); - exit (2); - } - } - - MODE = 0; - if ((N = get_argument (argc, argv, "-in"))) { - MODE = COPY_IN; - remove_argument (N, &argc, argv); - } - if ((N = get_argument (argc, argv, "-out"))) { - if (!MODE) { - fprintf (stderr, "ERROR: cannot use both -in and -out options!\n"); - usage_dvodist(); - } - MODE = COPY_OUT; - remove_argument (N, &argc, argv); - } - - if (argc != 2) usage(); - - return (TRUE); -} Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c (revision 33206) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/dvodist.c (revision 33207) @@ -1,3 +1,3 @@ -# include "photdbc.h" +# include "dvodist.h" int main (int argc, char **argv) { @@ -8,20 +8,37 @@ /* get configuration info, args, lockfile */ - initialize_dvodist (argc, argv); + initialize (argc, argv); + char *catdir = strcreate (argv[1]); + + LockDatabase (catdir); // dvodist (-out | -in) // dvodist -out : host table represents new target locations - hosts = HostTableLoad (); + HostTable *hosts = HostTableLoad (catdir); + if (NULL) { + Shutdown ("failed to load Host Table for %s\n", catdir); + } - // the output catalog needs to inherit the SKY_DEPTH of the input catalog - sky = SkyTableLoadOptimal (CATDIR, NULL, GSCFILE, TRUE, SKY_DEPTH_HST, VERBOSE); - SkyTableSetFilenames (sky, CATDIR, "cpt"); - skylist = SkyListByPatch (sky, -1, ®ION); + // load the current SkyTable. If SkyTable does not exist, we must fail + sky = SkyTableLoadOptimal (catdir, NULL, NULL, TRUE, SKY_DEPTH_HST, VERBOSE); + SkyTableSetFilenames (sky, catdir, "cpt"); + skylist = SkyListByPatch (sky, -1, &UserPatch); - AssignSkyToHost (skylist, hosts); + switch (MODE) { + case MODE_OUT: + AssignSkyToHost (skylist, hosts); + CopyToHostLocation (skylist, hosts); + break; + case MODE_IN: + CopyFromHostLocation (skylist, hosts); + break; + default: + fprintf (stderr, "impossible!"); + abort(); + } - CopyToHostLocation (skylist, hosts); - + // XXX probably need to make this a skyregion_io.c function + // char *SkyTableFilename (char *catdir); char *skyfile; ALLOCATE (skyfile, char, strlen(catdir) + strlen("/SkyTable.fits") + 1); @@ -31,2 +48,10 @@ exit (0); } + +/* things still missing AFAIK : + + * switch based on MODE + * add header keyword to SkyTable to note existence of hostID + * code for CopyFromHostLocation + + */ Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33207) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/initialize_dvodist.c (revision 33207) @@ -0,0 +1,81 @@ +# include "dvodist.h" + +void initialize (int argc, char **argv) { + + /* are these set correctly? */ + if (get_argument (argc, argv, "-h")) usage(); + if (get_argument (argc, argv, "--h")) usage(); + if (get_argument (argc, argv, "-help")) usage(); + if (get_argument (argc, argv, "--help")) usage(); + + // XXX anything? ConfigInit (&argc, argv); + args (argc, argv); +} + +void usage() { + + fprintf (stderr, "USAGE: dvodist (-out | -in) (catdir)\n\n"); + fprintf (stderr, " this program takes an existing DVO database and distributes it to the remote machine locations\n"); + + fprintf (stderr, " optional options:\n"); + fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -params : list the current parameters\n\n"); + + exit (2); +} + +int args (int argc, char **argv) { + + int N; + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + VERBOSE = TRUE; + remove_argument (N, &argc, argv); + } + + /* specify portion of the sky */ + UserPatch.Rmin = 0; + UserPatch.Rmax = 360; + UserPatch.Dmin = -90; + UserPatch.Dmax = +90; + if ((N = get_argument (argc, argv, "-region"))) { + remove_argument (N, &argc, argv); + UserPatch.Rmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Rmax = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmin = atof (argv[N]); + remove_argument (N, &argc, argv); + UserPatch.Dmax = atof (argv[N]); + remove_argument (N, &argc, argv); + + if (UserPatch.Rmin == UserPatch.Rmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Rmin == Rmax\n"); + exit (2); + } + if (UserPatch.Dmin == UserPatch.Dmax) { + fprintf (stderr, "ERROR: selected region is ill-defined: Dmin == Dmax\n"); + exit (2); + } + } + + MODE = MODE_NONE; + if ((N = get_argument (argc, argv, "-in"))) { + MODE = MODE_IN; + remove_argument (N, &argc, argv); + } + if ((N = get_argument (argc, argv, "-out"))) { + if (!MODE) { + fprintf (stderr, "ERROR: cannot use both -in and -out options!\n"); + usage(); + } + MODE = MODE_OUT; + remove_argument (N, &argc, argv); + } + if (!MODE) usage(); + + if (argc != 2) usage(); + + return (TRUE); +} Index: /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c =================================================================== --- /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c (revision 33207) +++ /branches/eam_branches/ipp-20111122/Ohana/src/photdbc/src/md5_ops.c (revision 33207) @@ -0,0 +1,45 @@ +# include "dvodist.h" + +// these are md5 operations that could probably be moved to libohana + +# define NBUFFER 512 + +// read a file, get the md5 sum, optionally write to 'output' +int get_md5_with_copy (char *input, char *output, md5_byte_t *digest) { + + // open the src file + FILE *fInput = fopen (input, "r"); + int fdInput = fileno (fInput); + + // open the tgt file + FILE *fOutput = NULL; + int fdOutput = 0; + if (output) { + fOutput = fopen (tgtname, "w"); + fdOutput = fileno (fOutput); + } + + int nbytes; + md5_state_t state; + md5_byte_t buffer[NBUFFER]; + + md5_init (&state); + while ((nbytes = read (fdInput, buffer, NBUFFER)) > 0) { + md5_append (&state, buffer, nbytes); + if (output) { + nbytes = write (fdOutput, buffer, NBUFFER); + // XXX check that we actually write out all nbytes... + } + } + md5_finish(&state, digest); + + fclose (fInput); + flush (fInput); + + if (output) { + fclose (fOutput); + flush (fOutput); + } + return TRUE; +} +