Index: /trunk/Ohana/src/opihi/cmd.data/queue2book.c
===================================================================
--- /trunk/Ohana/src/opihi/cmd.data/queue2book.c	(revision 42407)
+++ /trunk/Ohana/src/opihi/cmd.data/queue2book.c	(revision 42408)
@@ -1,27 +1,71 @@
 # include "pantasks.h"
 
-# define FREEKEYS \
-  if (keys) { for (i = 0; i < Nkeys; i++) { free (keys[i]); } free (keys); } \
-  for (i = 0; i < setWordN; i++) { free (setWordList[i]); free (setWordValue[i]); } \
+# define FREEKEYS							\
+  if (keys) { for (int ix = 0; ix < Nkeys; ix++) { free (keys[ix]); } free (keys); } \
+  for (int ix = 0; ix < setWordN; ix++) { free (setWordList[ix]); free (setWordValue[ix]); } \
   free (setWordValue); free (setWordList);
 
-// convert the named queue with ipptool metadata output into book format
+// USAGE: queue2book (queue) (book)
+// convert the named queue into book format
+// the output book will use the name supplied as the second argument
+// the input queue should either use the ippTools metadata format OR
+// the input queue should consist of a set of lines with white-space 
+// separated columns of data.  
+
+// **** MD format ****
+// In MD format, the queue starts with a line of the form "BOOKNAME MULTI"
+// following this, there will be a series of METADATA blocks.  the
+// value BOOKNAME in the first line much match the names of the METADATA blocks
+// note that the BOOKNAME is not actually used to set the output book name, 
+// and is not required to have any specific value other than matching the 
+// following METADATA blocks.
+
+// the METADATA blocks are used to create pages within the book (one page per block)
+// the key / value pairs within the METADATA blocks are used to set the values 
+// words within each page. 
+
+// option: -key (word[:word[:word..]])
+// pages are given names of the form 'page.NNN' unless -key options are supplied.
+// if one or more -key options are supplied, the value of the associated word is used
+// to set the pagename.  If multiple keys are supplied, the words are joined with colons.
+
+// option: -uniq
+// if -uniq is supplied, any new pages which match existing pages will be dropped / ignored
+
+// option: -setword
+// additional words may be added to the pages of the book when they are created.
+
+// **** Raw Column format ****
+
+// In Raw Column format, each line in the queue is converted to a new page. The white-space separated
+// words in each line are interpretted as the values of words on this page.  The names of these
+// columns (words) are specified on the first line of the queue.  Thus a valid input must consist of 
+// at least two lines.
+
+// we need these in both parsing blocks
+int Unique;
+int RawColumns;
+
+char **setWordList;
+char **setWordValue;
+int    setWordN;
+
+int   Nkeys;
+char **keys;
+
+int queue2book_rawcolumns (Queue *queue, Book *book);
+int queue2book_setpages (Book *book, Page *page);
+int queue2book_addwords (Book *book, Page *page);
+int queue2book_uniquify (Book *book, Page *page);
+
 int queue2book (int argc, char **argv) {
 
-  int i, N, onPage, found, Unique, Nkeys, NKEYS;
+  int N, onPage;
   char *line, *tmpword, *tmpvalue;
   char pagename[512]; // XXX this should be made dynamic, though it is an unlikey problem
-  char *bookName, **keys, *p, *q;
-  char **setWordList;
-  char **setWordValue;
-  int setWordN, setWordNalloc;
-
-  Page *page = NULL;
-  Book *book = NULL;
-  Queue *queue = NULL;
 
   /* supply additional constant words */
   setWordN = 0;
-  setWordNalloc = 10;
+  int setWordNalloc = 10;
   ALLOCATE (setWordList, char *, setWordNalloc);
   ALLOCATE (setWordValue, char *, setWordNalloc);
@@ -46,14 +90,22 @@
   }
 
+  RawColumns = FALSE;
+  if ((N = get_argument (argc, argv, "-raw-columns"))) {
+    remove_argument (N, &argc, argv);
+    RawColumns = TRUE;
+  }
+
+  // keys are used to define the page names (otherwise page.NNN is used)
   Nkeys = 0;
-  keys = NULL;
+  keys  = NULL;
   if ((N = get_argument (argc, argv, "-key"))) {
     remove_argument (N, &argc, argv);
 
     /* parse the key (word:word:word) into constituents */
-    NKEYS = 10;
+    int NKEYS = 10;
     ALLOCATE (keys, char *, NKEYS);
     
-    p = q = argv[N];
+    char *p = argv[N];
+    char *q = p;
     while (q != NULL) {
       q = strchr (p, ':');
@@ -67,5 +119,5 @@
 
   if (argc != 3) {
-    gprint (GP_ERR, "USAGE: queue2book (queue) (book) [-uniq] [-key key[:key..]]\n");
+    gprint (GP_ERR, "USAGE: queue2book (queue) (book) [-uniq] [-key key[:key..]] [-setword (word) (value)] [-raw-columns] \n");
     gprint (GP_ERR, "       converts the named queue  with ipptool metadata output into book format\n");
     FREEKEYS;
@@ -73,5 +125,5 @@
   }
 
-  queue = FindQueue (argv[1]);
+  Queue *queue = FindQueue (argv[1]);
   if (queue == NULL) {
     gprint (GP_ERR, "queue %s not found\n", argv[1]);
@@ -80,5 +132,10 @@
   }
     
-  book = CreateBook (argv[2]);
+  Book *book = CreateBook (argv[2]);
+
+  if (RawColumns) {
+    int status = queue2book_rawcolumns (queue, book);
+    return status;
+  }
 
   /* the first non-whitespace line of the queue should contain:
@@ -101,39 +158,32 @@
     free (line);
   }
-  bookName = thisword (line);
+  char *bookName = thisword (line);
   tmpword = nextword (line);
 
+  // check for BookName MULTI
   if (!tmpword || strcmp(tmpword, "MULTI")) {
     gprint (GP_ERR, "ERROR: missing metadata output name on first line\n");
-    free (bookName);
-    free (line);
-    FREEKEYS;
-    return (TRUE);
+    goto escape;
   }
   free (line);
   
+  // loop over lines, checking for new pages and words in the page
   onPage = FALSE;
+  Page *page = NULL;
   while (TRUE) {
     line = PopQueue (queue);
-    if (!line) {
+    if (!line) { // we have reached the end of the queue
+      if (onPage) { gprint (GP_ERR, "ERROR: unterminated metadata\n"); }
       free (bookName);
-      if (onPage) {
-	gprint (GP_ERR, "ERROR: unterminated metadata\n");
-	FREEKEYS;
-	return (TRUE);
-      }
       FREEKEYS;
       return (TRUE);
     }
     stripwhite (line);
-    if (!*line) {
-      free (line);
-      continue;
-    }
-    if (line[0] == '#') {
-      free (line);
-      continue;
-    }
+
+    // skip empty lines and commented-out lines
+    if (!*line) { free (line); continue; }
+    if (line[0] == '#') { free (line); continue; }
     
+    // parse the words on this line 
     tmpword = thisword (line);
     if (tmpword == NULL) abort(); // should not happen if line exists
@@ -142,9 +192,6 @@
       if (strcmp(tmpword, bookName)) {
 	gprint (GP_ERR, "ERROR: unexpected metadata name %s\n", tmpword);
-	free (line);
 	free (tmpword);
-	free (bookName);
-	FREEKEYS;
-	return (TRUE);
+	goto escape;
       }
       free (tmpword);
@@ -152,12 +199,10 @@
       if (strcmp(tmpword, "METADATA")) {
 	gprint (GP_ERR, "ERROR: missing METADATA tag\n");
-	free (line);
 	free (tmpword);
-	free (bookName);
-	FREEKEYS;
-	return TRUE;
+	goto escape;
       }
       onPage = TRUE;
-      // we don't know the page name until we load its data?
+      // we don't know the page name until we load its data, so set temporary name.
+      // this name will be used if no keys are supplied
       sprintf (pagename, "page.%03d", book[0].Npages);
       page = CreatePage (book, pagename);
@@ -167,5 +212,7 @@
     } 
 
-    /* close out the page */
+    // if we are here, we are within the page
+
+    // if we have hit the end of the page, we have some special work to do
     if (!strcmp(tmpword, "END")) {
       // XXX set the page name based on the contents
@@ -173,41 +220,7 @@
       free (line);
 
-      if (keys) {
-	memset (pagename, 0, 512);
-	for (i = 0; i < Nkeys; i++) {
-	  tmpword = BookGetWord (page, keys[i]);
-	  if (tmpword == NULL) {
-	    gprint (GP_ERR, "ERROR: missing key %s for ID\n", keys[i]);
-	    FREEKEYS;
-	    return (TRUE);
-	  }
-	  if (i > 0) strcat (pagename, ":");
-	  strcat (pagename, tmpword);
-	}
-	free (page[0].name);
-	page[0].name = strcreate (pagename);
-      }
-
-      // add the fixed words to the page
-      for (i = 0; i < setWordN; i++) {
-	BookSetWord (page, setWordList[i], setWordValue[i]);
-      }
-
-      if (Unique) {
-	/* search for an existing page with this name
-	   delete this one if one is found */
-	/* XXX don't attach this page to the book until this step is passed */	
-
-	found = FALSE;
-	for (i = 0; !found && (i < book[0].Npages); i++) {
-	  if (book[0].pages[i] == page) continue;
-	  if (!strcmp (book[0].pages[i][0].name, page[0].name)) {
-	    found = TRUE;
-	  }
-	}
-	if (found) {
-	  DeletePage (book, page);
-	}
-      }
+      if (!queue2book_setpages (book, page)) return (FALSE);
+      queue2book_addwords (book, page);
+      queue2book_uniquify (book, page);
 
       onPage = FALSE;
@@ -219,8 +232,5 @@
       gprint (GP_ERR, "ERROR: missing value for %s\n", tmpword);
       free (tmpword);
-      free (line);
-      free (bookName);
-      FREEKEYS;
-      return (TRUE);
+      goto escape;
     }
     BookSetWord (page, tmpword, tmpvalue);
@@ -231,18 +241,143 @@
   FREEKEYS;
   return (TRUE);
-}
+
+escape:
+  free (line);
+  free (bookName);
+  FREEKEYS;
+  return (TRUE);
+}
+
+int queue2book_rawcolumns (Queue *queue, Book *book) {
+
+  Page *idxPage = NULL;
+  char pagename[512];
+
+  while (TRUE) {
+    char *line = PopQueue (queue);
+    if (!line) { // we have reached the end of the queue
+      FREEKEYS;
+      if (idxPage) { DeletePage (book, idxPage); }
+      return (TRUE);
+    }
+    stripwhite (line);
+
+    // skip empty lines and commented-out lines
+    if (!*line) { free (line); continue; }
+    if (line[0] == '#') { free (line); continue; }
+    
+    Page *page = NULL;
+
+    // the first non-whitespace line of the queue should contain the column names
+    // save those words as the words on the index page
+
+    if (idxPage) {
+      sprintf (pagename, "page.%03d", book->Npages);
+    } else {
+      sprintf (pagename, "page.index");
+    }
+    page = CreatePage (book, pagename);
+
+    int Nw = 0; // index into the number of the word on the line
+    char *ptr = line;
+    while (ptr) {
+      if (idxPage && (Nw >= idxPage->Nwords)) {
+	gprint (GP_ERR, "ERROR: line has too many items: %s\n", line);
+	return (FALSE);
+      }
+	
+      char *tmpword = thisword (ptr);
+      if (tmpword == NULL) abort(); // should not happen if ptr is not NULL
+
+      // need to use the index page to get the words
+      if (idxPage) {
+	BookSetWord (page, idxPage->words[Nw], tmpword);
+      } else {
+	BookSetWord (page, tmpword, "index");
+      }
+
+      free (tmpword);
+
+      ptr = nextword (ptr);
+      Nw ++;
+    }
+    if (!queue2book_setpages (book, page)) return (FALSE);
+    queue2book_addwords (book, page);
+    queue2book_uniquify (book, page);
+
+    if (!idxPage) { idxPage = page; }
+  }
+  return FALSE; // it should not be possible to reach here
+}
+
+// keys, Nkeys are file-globals
+int queue2book_setpages (Book *book, Page *page) {
+
+  char pagename[512];
+
+  // set the page name based on the key values
+  if (!keys) return TRUE;
+
+  memset (pagename, 0, 512);
+  for (int i = 0; i < Nkeys; i++) {
+    char *tmpword = BookGetWord (page, keys[i]);
+    if (tmpword == NULL) {
+      gprint (GP_ERR, "ERROR: missing key %s for ID\n", keys[i]);
+      FREEKEYS;
+      return FALSE;
+    }
+    if (i > 0) strcat (pagename, ":");
+    strcat (pagename, tmpword);
+  }
+  free (page[0].name);
+  page[0].name = strcreate (pagename);
+  return TRUE;
+}
+
+// setWordList, setWordValue, setWordN are file-globals
+int queue2book_addwords (Book *book, Page *page) {
+  // add the fixed words to the page
+  for (int i = 0; i < setWordN; i++) {
+    BookSetWord (page, setWordList[i], setWordValue[i]);
+  }
+  return TRUE;
+}
+
+// setWordList, setWordValue, setWordN are file-globals
+// it would be cleaner to not attach this page to the book until this step is passed
+int queue2book_uniquify (Book *book, Page *page) {
+
+  if (!Unique) return TRUE;
+
+  // search for an existing page with this name
+  // delete this one if one is found 
+  
+  int found = FALSE;
+  for (int i = 0; !found && (i < book->Npages); i++) {
+    if (book->pages[i] == page) continue;
+    if (!strcmp (book->pages[i]->name, page->name)) {
+      found = TRUE;
+    }
+  }
+  if (found) {
+    DeletePage (book, page);
+  }
+  return TRUE;
+}
+
+
 
 /* scan through the queue lines.  these should look like:  
 
-bookName METADATA
-key type value
-key type value
-...
-END
-
-bookName METADATA
-key type value
-key type value
-...
-END
+   bookName METADATA
+   key type value
+   key type value
+   ...
+   END
+
+   bookName METADATA
+   key type value
+   key type value
+   ...
+   END
 */