Index: /branches/eam_branches/ipp-20150625/Ohana/src/tools/Makefile =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/tools/Makefile (revision 38966) +++ /branches/eam_branches/ipp-20150625/Ohana/src/tools/Makefile (revision 38967) @@ -17,5 +17,5 @@ PROGRAMS = gconfig fhead pspsconvert ftable fields list_astro glockfile \ radec mktemp precess csystem fits_insert \ -medianfilter mefhead ckfits roc random +medianfilter mefhead ckfits roc random fits_to_mysql all tools: $(PROGRAMS) Index: /branches/eam_branches/ipp-20150625/Ohana/src/tools/src/fits_to_mysql.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/tools/src/fits_to_mysql.c (revision 38966) +++ /branches/eam_branches/ipp-20150625/Ohana/src/tools/src/fits_to_mysql.c (revision 38967) @@ -2,346 +2,5 @@ # include # include "inttypes.h" - -# define NMAX 0x100 -# define NBUFFER 0x100000 -# define NROWS 10000 - -# define TO_BUFFER 1 - -char *DATABASE_HOST; -char *DATABASE_USER; -char *DATABASE_PASS; -char *DATABASE_NAME; - -int args (int *argc, char **argv); -void usage(); - -void print_data (FTable *table); -void *gfits_get_bintable_column_data_by_Ncol (FTable *table, int Nfield, char *type, off_t *Nrow, int *Ncol, char nativeOrder); -void FFPRINT (char *buffer, int *nbuffer, double value); - -int main (int argc, char **argv) { - - args (&argc, argv); - if (argc != 2) usage (); - char *input = argv[1]; - - FTable table; - Header header; - table.header = &header; - table.buffer = NULL; - - FILE *f = fopen (input, "r"); - if (f == (FILE *) NULL) { - fprintf (stderr, "can't open file %s\n", input); - exit (1); - } - - int Nsection = 0; - while (gfits_fread_header (f, &header)) { - - /* extract the EXTNAME for this component (set to PHU for 0th component) */ - char extname[80]; - int status = gfits_scan (&header, "EXTNAME", "%s", 1, extname); - if (!status) { - if (Nsection == 0) { - strcpy (extname, "PHU"); - } else { - strcpy (extname, "UNKNOWN"); - } - } - fprintf (stdout, "%-30s ", extname); - - /* extract the datatype for this component (IMAGE for 0th component) */ - char exttype[80]; - if (Nsection == 0) { - strcpy (exttype, "IMAGE"); - } else { - status = gfits_scan (&header, "XTENSION", "%s", 1, exttype); - if (!status) { - strcpy (exttype, "UNKNOWN"); - } - } - fprintf (stdout, "%-15s ", exttype); - - fprintf (stdout, " %4d", header.Naxes); - - /* extract the individual axes */ - int i; - for (i = 0; i < header.Naxes; i++) { - fprintf (stdout, " "OFF_T_FMT, header.Naxis[i]); - } - fprintf (stdout, "\n"); - - if (strcmp(exttype, "BINTABLE")) { - off_t Nbytes = gfits_data_size (&header); - gfits_free_header (&header); - fseeko (f, Nbytes, SEEK_CUR); - Nsection ++; - continue; - } - - IOBuffer insert; - InitIOBuffer (&insert, 1024); - - create_table (&header. extname, &insert); - - // need to reset this on each pass? - myAssert (!table.buffer, "oops"); - - // total number of bytes in the data section (including pad at the end) - off_t Nbytes = gfits_data_size (&header); - - off_t Ntotal = header.Naxis[1]; - int Nrows = NROWS; - int start = 0; - while (start < Ntotal) { - if (!gfits_fread_ftable_range (f, FALSE, TRUE, &table, start, Nrows)) { - fprintf (stderr, "error reading table for extension %d\n", Nsection); - exit (1); - } - fprintf (stderr, "read "OFF_T_FMT" rows\n", header.Naxis[1]); - - write_data (&table, &insert); - - start += Nrows; - Nbytes -= header.Naxis[0]*header.Naxis[1]*abs(header.bitpix / 8); - header.Naxis[1] = Ntotal; - } - - gfits_free_table (&table); - gfits_free_header (&header); - - // move the pointer to the end of this data segment - fseeko (f, Nbytes, SEEK_CUR); - Nsection ++; - - FreeIOBuffer (&insert); - } - gfits_free_header (&header); - - fclose (f); - - ohana_memdump (TRUE); - - exit (0); -} - -// the 'insert' buffer will be used to generate the insert lines downstream (call InitIOBuffer on it first) -int create_tables (Header *header, char *extname, IOBuffer *insert) { - - int i, j, Nf; - - // print out the columns (need to swap first) - - int Nfields; - if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE; - - IOBuffer buffer; - InitIOBuffer (&buffer, 1024); - - // drop the table if it exists - PrintIOBuffer (&buffer, "DROP TABLE if exists %s\n", extname); - status = mysql_query(mysql, buffer.buffer); - if (status) { - fprintf (stderr, "failed to drop table:\n"); - fprintf (stderr, "%s\n", mysql_error(mysql)); - return FALSE; - } - buffer.Nbuffer = 0; - bzero (buffer.buffer, buffer.Nalloc); - - // Only send the necessary fields (eg, do not sent parallax and pm) - PrintIOBuffer (&buffer, "CREATE TABLE %s (\n", extname); - PrintIOBuffer (&buffer, "INSERT INTO %s (\n", extname); - - // first, extract the table data: - for (i = 0; i < Nfields; i++) { - - snprintf (field, 256, "TTYPE%d", i); - gfits_scan (header, field, "%s", 1, tlabel); - - snprintf (field, 256, "TFORM%d", N); - gfits_scan (header, field, "%s", 1, format); - - char type[16]; - if (!gfits_bintable_format (format, type, &Nval, &Nbytes)) return (NULL); - - if (!strcmp (type, "byte")) { - PrintIOBuffer (&buffer, "%s TINYINT", tlabel); - } - if (!strcmp (type, "short")) { - PrintIOBuffer (&buffer, "%s SMALLINT", tlabel); - } - if (!strcmp (type, "int")) { - PrintIOBuffer (&buffer, "%s INT", tlabel); - } - if (!strcmp (type, "int64_t")) { - PrintIOBuffer (&buffer, "%s BITINT", tlabel); - } - if (!strcmp (type, "float")) { - PrintIOBuffer (&buffer, "%s FLOAT", tlabel); - } - if (!strcmp (type, "double")) { - PrintIOBuffer (&buffer, "%s DOUBLE", tlabel); - } - if (!strcmp (type, "char")) { - PrintIOBuffer (&buffer, "%s VARCHAR(%d)", tlabel, Nval); - } - - PrintIOBuffer (insert, "%s", tlabel); - if (i == Nfields - 1) { - PrintIOBuffer (&buffer, ")\n"); - PrintIOBuffer (insert, ") VALUES \n"); - } else { - PrintIOBuffer (&buffer, ",\n"); - PrintIOBuffer (insert, ",\n"); - } - } - - if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); - status = mysql_query(mysql, buffer.buffer); - if (status) { - fprintf (stderr, "failed to create table:\n"); - fprintf (stderr, "%s\n", mysql_error(mysql)); - fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); - } - - FreeIOBuffer (&buffer); - return TRUE; -} - -typedef enum { - TYPE_NONE, - TYPE_CHAR, - TYPE_BYTE, - TYPE_SHORT, - TYPE_INT, - TYPE_INT64, - TYPE_FLOAT, - TYPE_DOUBLE -} TypeInt; - -void write_data (FTable *table, IOBuffer *insert) { - - int i, j, Nf; - - // copy the 'insert' IOBuffer to the output buffer - IOBuffer output; - InitIOBuffer (&output, 1024); - IOBufferCopy (&output, &insert); - - // print out the columns (need to swap first) - - Header *header = table->header; - int Ny = table->header->Naxis[1]; - - int Nfields; - if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return; - - ALLOCATE_PTR (Nrow, off_t, Nfields); - ALLOCATE_PTR (Ncol, int, Nfields); - ALLOCATE_PTR (data, void *, Nfields); - ALLOCATE_PTR (type, char *, Nfields); - ALLOCATE_PTR (Type, int, Nfields); - - // first, extract the table data: (probably should save this structure and re-use in create_table) - for (i = 0; i < Nfields; i++) { - ALLOCATE (type[i], char, 16); - data[i] = gfits_get_bintable_column_data_by_Ncol (table, i + 1, type[i], &Nrow[i], &Ncol[i], FALSE); - myAssert (Nrow[i] == Ny, "oops"); - Type[i] = TYPE_NONE; - if (!strcmp (type[i], "char")) Type[i] = TYPE_CHAR; - if (!strcmp (type[i], "byte")) Type[i] = TYPE_BYTE; - if (!strcmp (type[i], "short")) Type[i] = TYPE_SHORT; - if (!strcmp (type[i], "int")) Type[i] = TYPE_INT; - if (!strcmp (type[i], "int64_t")) Type[i] = TYPE_INT64; - if (!strcmp (type[i], "float")) Type[i] = TYPE_FLOAT; - if (!strcmp (type[i], "double")) Type[i] = TYPE_DOUBLE; - myAssert (Type[i] != TYPE_NONE, "oops"); - } - - // convert the type characters to an enum for speed here - // could also generate pointers of all types to the data; - - for (i = 0; i < Ny; i++) { - PrintIOBuffer (output, "( "); - for (Nf = 0; Nf < Nfields; Nf++) { - switch (Type[Nf]) { - case TYPE_CHAR: { - char line[1024]; - char *value = data[Nf]; - memset (line, 0, 128); - memcpy (line, &value[i*Ncol[Nf]], Ncol[Nf]); - PrintIOBuffer (output, "%s", line); - break; - } - case TYPE_BYTE: { - char *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%hhd", value[i*Ncol[Nf] + j]); - } - break; - } - case TYPE_SHORT: { - short *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%hd", value[i*Ncol[Nf] + j]); - } - break; - } - case TYPE_INT: { - int *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%d", value[i*Ncol[Nf] + j]); - } - break; - } - case TYPE_INT64: { - int64_t *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%lld", (long long int) value[i*Ncol[Nf] + j]); - } - break; - } - case TYPE_FLOAT: { - float *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%e", value[i*Ncol[Nf] + j]); - } - break; - } - case TYPE_DOUBLE: { - double *value = data[Nf]; - for (j = 0; j < Ncol[Nf]; j++) { - PrintIOBuffer (output, "%e", value[i*Ncol[Nf] + j]); - } - break; - } - default: - myAbort ("impossible"); - } - if (Nf < Nfields - 1) { - PrintIOBuffer (output, ",\n"); - } else { - PrintIOBuffer (output, ")\n"); - } - if (output->Nbuffer > MAX_BUFFER) { - } - } - } - - for (i = 0 ; i < Nfields; i++) { - free (type[i]); - free (data[i]); - } - free (Nrow); - free (Ncol); - free (data); - free (type); - free (Type); - - return; -} +# include "mysql.h" # define SWAP_BYTE { \ @@ -359,4 +18,446 @@ tmp = Pin[3]; Pin[3] = Pin[4]; Pin[4] = tmp; } +# define DEBUG 0 +# define NMAX 0x100 +# define NBUFFER 0x100000 +# define NROWS 10000 +# define MAX_BUFFER 0x080000 + +# define TO_BUFFER 1 + +typedef enum { + TYPE_NONE, + TYPE_CHAR, + TYPE_BYTE, + TYPE_SHORT, + TYPE_INT, + TYPE_INT64, + TYPE_FLOAT, + TYPE_DOUBLE +} TypeInt; + +// generate a structure to describe the table: +typedef struct { + char *rawname; // name of the column in the fits header + char *colname; // name of the column to be used in the db + char *format; + TypeInt type; + int Nval; + int Nbytes; +} ColumnDef; + +char *DATABASE_HOST; +char *DATABASE_USER; +char *DATABASE_PASS; +char *DATABASE_NAME; + +int args (int *argc, char **argv); +void usage(); + +void FreeColumnDef (ColumnDef *columns, int Ncolumns); +ColumnDef *parse_table (Header *header, int *ncolumn); +int create_table (ColumnDef *columns, int Ncolumns, char *extname, IOBuffer *insert, MYSQL *mysql); +void write_data (ColumnDef *columns, int Ncolumns, FTable *table, IOBuffer *insert, MYSQL *mysql); + +void *gfits_get_bintable_column_data_by_Ncol (FTable *table, int Nfield, char *type, off_t *Nrow, int *Ncol, char nativeOrder); +int CopyIOBuffer (IOBuffer *output, IOBuffer *input); + +MYSQL *mysql_connect (MYSQL *mysqlBase); +int mysql_commit_buffer (IOBuffer *buffer, MYSQL *mysql); + +int main (int argc, char **argv) { + + args (&argc, argv); + if (argc != 2) usage (); + char *input = argv[1]; + + MYSQL mysqlBase; + MYSQL *mysqlReal = mysql_connect (&mysqlBase); + if (!mysqlReal) { + fprintf (stderr, "failed to connect to mysql\n"); + exit (1); + } + + FTable table; + Header header; + table.header = &header; + table.buffer = NULL; + + FILE *f = fopen (input, "r"); + if (f == (FILE *) NULL) { + fprintf (stderr, "can't open file %s\n", input); + exit (1); + } + + int Nsection = 0; + while (gfits_fread_header (f, &header)) { + + /* extract the EXTNAME for this component (set to PHU for 0th component) */ + char extname[80]; + int status = gfits_scan (&header, "EXTNAME", "%s", 1, extname); + if (!status) { + if (Nsection == 0) { + strcpy (extname, "PHU"); + } else { + strcpy (extname, "UNKNOWN"); + } + } + fprintf (stdout, "%-30s ", extname); + + /* extract the datatype for this component (IMAGE for 0th component) */ + char exttype[80]; + if (Nsection == 0) { + strcpy (exttype, "IMAGE"); + } else { + status = gfits_scan (&header, "XTENSION", "%s", 1, exttype); + if (!status) { + strcpy (exttype, "UNKNOWN"); + } + } + fprintf (stdout, "%-15s ", exttype); + + fprintf (stdout, " %4d", header.Naxes); + + /* extract the individual axes */ + int i; + for (i = 0; i < header.Naxes; i++) { + fprintf (stdout, " "OFF_T_FMT, header.Naxis[i]); + } + fprintf (stdout, "\n"); + + if (strcmp(exttype, "BINTABLE")) { + off_t Nbytes = gfits_data_size (&header); + gfits_free_header (&header); + fseeko (f, Nbytes, SEEK_CUR); + Nsection ++; + continue; + } + + int Ncolumns; + ColumnDef *columns = parse_table (&header, &Ncolumns); + + IOBuffer insert; + InitIOBuffer (&insert, 1024); + + create_table (columns, Ncolumns, extname, &insert, mysqlReal); + + // need to reset this on each pass? + myAssert (!table.buffer, "oops"); + + // total number of bytes in the data section (including pad at the end) + off_t Nbytes = gfits_data_size (&header); + + off_t Ntotal = header.Naxis[1]; + int Nrows = NROWS; + int start = 0; + while (start < Ntotal) { + if (!gfits_fread_ftable_range (f, FALSE, TRUE, &table, start, Nrows)) { + fprintf (stderr, "error reading table for extension %d\n", Nsection); + exit (1); + } + fprintf (stderr, "read "OFF_T_FMT" rows\n", header.Naxis[1]); + + write_data (columns, Ncolumns, &table, &insert, mysqlReal); + + start += Nrows; + Nbytes -= header.Naxis[0]*header.Naxis[1]*abs(header.bitpix / 8); + header.Naxis[1] = Ntotal; + } + + gfits_free_table (&table); + gfits_free_header (&header); + + // move the pointer to the end of this data segment + fseeko (f, Nbytes, SEEK_CUR); + Nsection ++; + + FreeColumnDef (columns, Ncolumns); + FreeIOBuffer (&insert); + } + gfits_free_header (&header); + + fclose (f); + + FREE (DATABASE_HOST); + FREE (DATABASE_USER); + FREE (DATABASE_PASS); + FREE (DATABASE_NAME); + + ohana_memdump (TRUE); + + exit (0); +} + +void FreeColumnDef (ColumnDef *columns, int Ncolumns) { + + int i; + + if (!columns) return; + for (i = 0; i < Ncolumns; i++) { + FREE (columns[i].rawname); + FREE (columns[i].colname); + FREE (columns[i].format); + } + FREE (columns); + return; +} + +ColumnDef *parse_table (Header *header, int *ncolumn) { + + int i, j; + + int Ncolumn = 0; + int NCOLUMN = 50; + ColumnDef *column = NULL; + ALLOCATE (column, ColumnDef, NCOLUMN); + + // print out the columns (need to swap first) + + int Nfields; + if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return NULL; + + // first, extract the table data: + for (i = 0; i < Nfields; i++) { + + char field[256], tlabel[256], format[256]; + snprintf (field, 256, "TTYPE%d", i+1); + gfits_scan (header, field, "%s", 1, tlabel); + + snprintf (field, 256, "TFORM%d", i+1); + gfits_scan (header, field, "%s", 1, format); + + // check for duplicates + int Ndup = 0; + for (j = 0; j < Ncolumn; j++) { + if (strcmp (column[j].rawname, tlabel)) continue; + Ndup ++; + } + if (Ndup) { + char tmpname[256]; + snprintf (tmpname, 256, "%s_%d", tlabel, Ndup); + column[Ncolumn].colname = strcreate (tmpname); + } else { + column[Ncolumn].colname = strcreate (tlabel); + } + column[Ncolumn].rawname = strcreate (tlabel); + column[Ncolumn].format = strcreate (format); + + char type[16]; + int Nval, Nbytes; + if (!gfits_bintable_format (format, type, &Nval, &Nbytes)) return NULL; + column[Ncolumn].Nval = Nval; + column[Ncolumn].Nbytes = Nbytes; + + // save the type as well + if (!strcmp (type, "byte")) column[Ncolumn].type = TYPE_BYTE; + if (!strcmp (type, "short")) column[Ncolumn].type = TYPE_SHORT; + if (!strcmp (type, "int")) column[Ncolumn].type = TYPE_INT; + if (!strcmp (type, "int64_t")) column[Ncolumn].type = TYPE_INT64; + if (!strcmp (type, "float")) column[Ncolumn].type = TYPE_FLOAT; + if (!strcmp (type, "double")) column[Ncolumn].type = TYPE_DOUBLE; + if (!strcmp (type, "char")) column[Ncolumn].type = TYPE_CHAR; + + Ncolumn ++; + if (Ncolumn == NCOLUMN) { + NCOLUMN += 100; + REALLOCATE (column, ColumnDef, NCOLUMN); + } + } + *ncolumn = Ncolumn; + return column; +} + +// the 'insert' buffer will be used to generate the insert lines downstream (call InitIOBuffer on it first) +int create_table (ColumnDef *columns, int Ncolumns, char *extname, IOBuffer *insert, MYSQL *mysql) { + + int i; + + IOBuffer buffer; + InitIOBuffer (&buffer, 1024); + + char *tablename = strcreate (extname); + + // convert "." in tablename to _ + char *p = tablename; + while (p && *p) { + if (*p == '.') *p = '_'; + p++; + } + + // drop the table if it exists + PrintIOBuffer (&buffer, "DROP TABLE if exists %s\n", tablename); + int status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to drop table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + return FALSE; + } + buffer.Nbuffer = 0; + bzero (buffer.buffer, buffer.Nalloc); + + // Only send the necessary fields (eg, do not sent parallax and pm) + PrintIOBuffer (&buffer, "CREATE TABLE %s (\n", tablename); + PrintIOBuffer (insert, "INSERT INTO %s (\n", tablename); + + // first, extract the table data: + for (i = 0; i < Ncolumns; i++) { + + // watch out for Nval != 1 + switch (columns[i].type) { + case TYPE_BYTE: + PrintIOBuffer (&buffer, "%s TINYINT", columns[i].colname); + break; + case TYPE_SHORT: + PrintIOBuffer (&buffer, "%s SMALLINT", columns[i].colname); + break; + case TYPE_INT: + PrintIOBuffer (&buffer, "%s INT", columns[i].colname); + break; + case TYPE_INT64: + PrintIOBuffer (&buffer, "%s BIGINT", columns[i].colname); + break; + case TYPE_FLOAT: + PrintIOBuffer (&buffer, "%s FLOAT", columns[i].colname); + break; + case TYPE_DOUBLE: + PrintIOBuffer (&buffer, "%s DOUBLE", columns[i].colname); + break; + case TYPE_CHAR: + PrintIOBuffer (&buffer, "%s VARCHAR(%d)", columns[i].colname, columns[i].Nval); + break; + default: + myAbort ("oops"); + } + + PrintIOBuffer (insert, "%s", columns[i].colname); + if (i == Ncolumns - 1) { + PrintIOBuffer (&buffer, ")\n"); + PrintIOBuffer (insert, ") VALUES \n"); + } else { + PrintIOBuffer (&buffer, ",\n"); + PrintIOBuffer (insert, ",\n"); + } + } + + if (DEBUG) fprintf (stderr, "%s\n", buffer.buffer); + status = mysql_query(mysql, buffer.buffer); + if (status) { + fprintf (stderr, "failed to create table:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer.Nbuffer); + } + + FreeIOBuffer (&buffer); + free (tablename); + + return TRUE; +} + +void write_data (ColumnDef *columns, int Ncolumns, FTable *table, IOBuffer *insert, MYSQL *mysql) { + + int i, j, Nf; + + // copy the 'insert' IOBuffer to the output buffer + IOBuffer output; + InitIOBuffer (&output, 1024); + CopyIOBuffer (&output, insert); + + int Ny = table->header->Naxis[1]; + + ALLOCATE_PTR (data, void *, Ncolumns); + + // first, extract the table data: (probably should save this structure and re-use in create_table) + for (i = 0; i < Ncolumns; i++) { + off_t Nrow; + int Ncol; + char type[16]; + data[i] = gfits_get_bintable_column_data_by_Ncol (table, i + 1, type, &Nrow, &Ncol, FALSE); + myAssert (Nrow == Ny, "oops"); + } + + // convert the type characters to an enum for speed here + // could also generate pointers of all types to the data; + + for (i = 0; i < Ny; i++) { + PrintIOBuffer (&output, "( "); + for (Nf = 0; Nf < Ncolumns; Nf++) { + switch (columns[Nf].type) { + case TYPE_CHAR: { + char line[1024]; + char *value = data[Nf]; + memset (line, 0, 128); + memcpy (line, &value[i*columns[Nf].Nval], columns[Nf].Nval); + PrintIOBuffer (&output, "%s", line); + break; + } + case TYPE_BYTE: { + char *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%hhd", value[i*columns[Nf].Nval + j]); + } + break; + } + case TYPE_SHORT: { + short *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%hd", value[i*columns[Nf].Nval + j]); + } + break; + } + case TYPE_INT: { + int *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%d", value[i*columns[Nf].Nval + j]); + } + break; + } + case TYPE_INT64: { + int64_t *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%lld", (long long int) value[i*columns[Nf].Nval + j]); + } + break; + } + case TYPE_FLOAT: { + float *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%e", value[i*columns[Nf].Nval + j]); + } + break; + } + case TYPE_DOUBLE: { + double *value = data[Nf]; + for (j = 0; j < columns[Nf].Nval; j++) { + PrintIOBuffer (&output, "%e", value[i*columns[Nf].Nval + j]); + } + break; + } + default: + myAbort ("impossible"); + } + if (Nf < Ncolumns - 1) { + PrintIOBuffer (&output, ",\n"); + } else { + PrintIOBuffer (&output, "),\n"); + } + } + if (output.Nbuffer > MAX_BUFFER) { + if (DEBUG) fprintf (stderr, "%s\n", output.buffer); + mysql_commit_buffer (&output, mysql); + CopyIOBuffer (&output, insert); + } + } + + if (DEBUG) fprintf (stderr, "%s\n", output.buffer); + mysql_commit_buffer (&output, mysql); + FreeIOBuffer (&output); + + for (i = 0 ; i < Ncolumns; i++) { + free (data[i]); + } + free (data); + return; +} + void *gfits_get_bintable_column_data_by_Ncol (FTable *table, int Nfield, char *type, off_t *Nrow, int *Ncol, char nativeOrder) { @@ -523,4 +624,5 @@ } else usage(); + return TRUE; } @@ -534,2 +636,109 @@ exit (2); } + +int dump_result (MYSQL *connection) { + + MYSQL_RES *result = mysql_store_result (connection); + if (!result) return FALSE; + int Nrows = mysql_num_rows(result); + int Ncols = mysql_num_fields(result); + + int i, j; + for (j = 0; j < Nrows; j++) { + MYSQL_ROW row = mysql_fetch_row(result); + for (i = 0; i < Ncols; i++) { + fprintf (stderr, "%s ", row[i]); + } + fprintf (stderr, "\n"); + } + mysql_free_result (result); + return (TRUE); +} + +// DATABASE_* are supplied on the command line and are global variables in dvopsps.h +MYSQL *mysql_connect (MYSQL *mysqlBase) { + + char query[256]; + + mysql_init (mysqlBase); + MYSQL *connection = mysql_real_connect (mysqlBase, DATABASE_HOST, DATABASE_USER, DATABASE_PASS, DATABASE_NAME, 0, 0, 0); + + if (connection == NULL) { + fprintf (stderr, "failed to connect to database\n"); + fprintf (stderr, "%s\n", mysql_error (mysqlBase)); + return NULL; + } + + sprintf (query, "set @@interactive_timeout = 30000;"); + if (mysql_query (connection, query)) { + fprintf (stderr, "failed to set interactive timout\n"); + fprintf (stderr, "%s\n", mysql_error (connection)); + return NULL; + } + dump_result (connection); + + sprintf (query, "set @@wait_timeout = 30000;"); + if (mysql_query (connection, query)) { + fprintf (stderr, "failed to set wait timout\n"); + fprintf (stderr, "%s\n", mysql_error (connection)); + return NULL; + } + dump_result (connection); + + // sprintf (query, "set max_allowed_packet = %d;", 2*MAX_BUFFER); + // if (mysql_query (connection, query)) { + // fprintf (stderr, "failed to set max_allowed_packet\n"); + // fprintf (stderr, "%s\n", mysql_error (connection)); + // return NULL; + // } + // dump_result (connection); + + // sprintf (query, "show variables like 'max_allowed_packet';"); + // if (mysql_query (connection, query)) { + // fprintf (stderr, "failed to set max_allowed_packet\n"); + // fprintf (stderr, "%s\n", mysql_error (connection)); + // return NULL; + // } + // dump_result (connection); + + if (0) { + sprintf (query, "set autocommit=0;"); + if (mysql_query (connection, query)) { + fprintf (stderr, "failed to turn off autocommit\n"); + fprintf (stderr, "%s\n", mysql_error (connection)); + return NULL; + } + dump_result (connection); + } + + return connection; +} + +int mysql_commit_buffer (IOBuffer *buffer, MYSQL *mysql) { + + MYSQL_RES *result; + + // check that the last two chars are ,\n and replace with ;\n + if (!strcmp(&buffer->buffer[buffer->Nbuffer-2], ",\n")) { + buffer->buffer[buffer->Nbuffer-2] = ';'; + } else { + fprintf (stderr, "invalid sql?\n"); + return FALSE; + } + + // XXX check return status + if (mysql) { + int status; + if (DEBUG) fprintf (stderr, "%s\n", buffer->buffer); + status = mysql_query(mysql, buffer->buffer); + if (status) { + fprintf (stderr, "error with insert:\n"); + fprintf (stderr, "%s\n", mysql_error(mysql)); + fprintf (stderr, "Nbuffer: %d\n", buffer->Nbuffer); + } + result = mysql_store_result (mysql); + if (result) mysql_free_result (result); + } + + return TRUE; +}