Index: /trunk/doc/draft/metadata.tex =================================================================== --- /trunk/doc/draft/metadata.tex (revision 178) +++ /trunk/doc/draft/metadata.tex (revision 179) @@ -59,5 +59,6 @@ \section{Metadata Representation} -We propose that an item of metadata be represented as a C structure: +We propose that an item of metadata be represented as a C structure with at least the following +fields: \begin{verbatim} /* @@ -75,6 +76,5 @@ } val; // value of metadata char *comment; // optional comment ("", not NULL) - -} psMetaData; +} psMetaDataItem; \end{verbatim} @@ -89,5 +89,5 @@ The possible types of metadata are identified by the enumerated type -\code{psMetaDataType} (see also table \ref{tabMetaDataTypes}): +\code{psMetaDataType} (see also table \ref{tabMetaDataTypes}); the initial defined values are: \begin{verbatim} /* @@ -95,5 +95,5 @@ */ typedef enum { // type of val is: - psMetaFloat = 0, // float (.f) + psMetaFloat, // float (.f) psMetaInt, // int (.i) psMetaStr, // string (.v) @@ -129,31 +129,51 @@ \section{Collections of Metadata} -We also need to support a container class for metadata. We propose -using the standard \PS{} doubly-linked list type \code{psDlist} -(see \href{file:utils#psDlist}{utils.pdf} for details). For example: -\begin{verbatim} - psDlist *list = psDlistNew(NULL); // list of metadata - - psDlistAppend(list, psMetaDataNew("const.ten", psMetaInt, &ten, NULL, 0)); - psDlistAppend(list, psMetaDataNew("const.sqrt2", psMetaFloat, &sqrt2, - "square root of two", 0)); - psDlistAppend(list, psMetaDataNew("lang.hello", psMetaStr, "Bonjour", - "French", 0)); - /* - * Print all metadata to stdout - */ - (void)psDlistGet(list, psDlistTail); // initialise iterator - while ((ms = psDlistGet(list, psDlistPrev)) != NULL) { - psMetaDataPrint(stdout, ms); - } - /* - * Clean up - */ - psDlistDel(list, (void (*)(void *))psMetaDataDel); +\begin{verbatim} +typedef struct { + psDlist *list; // list of psMetaDataItem + psHash *table; // hash table of the same metadata +} psMetaDataSet; +\end{verbatim} + +The type \code{psMetaDataSet} is a container class for metadata. Note that there are +in fact \emph{two} representations of the metadata (each \code{psMetaDataItem} appears +on both). + +We are using the standard \PS{} doubly-linked list types \code{psDlist} and \code{psHash} +(see \href{file:utils#psDlist}{utils.pdf:psDlist} and \href{file:utils#psHash}{utils.pdf:psHash} for details). +For example: +\begin{verbatim} + for (int i = 0; i < 10; i += 5) { + float sqrti = sqrt(i); + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_INT, &i, NULL, "const.%d", i)); + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_FLOAT, &sqrti, "square root", "const.sqrt%d", i)); + } + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_STR, "Bonjour", "French", "lang.hello")); + /* + * Remove a key + */ + psMetaDataItemFree(psMetaDataRemove(ms, "lang.hello")); + /* + * Print all metadata + */ + fprintf(stdout, "------\n"); + psMetaDataSetIterator(ms); + while ((meta = psMetaDataGetNext(ms, NULL)) != NULL) { + psMetaDataItemPrint(stdout, meta, ""); + } + /* + * Look up a key by name + */ + fprintf(stdout, "------\n"); + fprintf(stdout, "Looking up by name:\n"); + meta = psMetaDataLookup(ms, "const.5"); + if (meta != NULL) { + psMetaDataItemPrint(stdout, meta, ""); + } \end{verbatim} \section{Naming convention for MetaData} -The \code{psMetaData} struct includes a name. This name should be of +The \code{psMetaDataItem} struct includes a name. This name should be of the form \code{name1.name2.name3$\cdots$}, e.g.\hfil\break \null\qquad\qquad\code{IPP.phase1.ota12.biassec}. @@ -163,4 +183,64 @@ all metadata names in use throughout the project. +\subsection{Support for Multiple Values for a Given Name} + +\begin{verbatim} + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_STR | PS_META_NON_UNIQUE, + "Bonjour", "French", "lang.hello")); + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_STR | PS_META_NON_UNIQUE, + "Aloha", "Hawaiian", "lang.hello")); + psMetaDataAppend(ms, psMetaDataItemAlloc(PS_META_STR | PS_META_NON_UNIQUE, + "Good Morning", "English", "lang.hello")); + /* + * Print all metadata starting "lang" + */ + psMetaDataSetIterator(ms); + while ((meta = psMetaDataGetNext(ms, "lang")) != NULL) { + psMetaDataItemPrint(stdout, meta, ""); + } +\end{verbatim} + +It is an unfortunate fact that certain metadata keywords (such as \code{COMMENT} in a FITS header) +may be repeated with different values. The \code{psMetaDataAppend} routine is required +to check that all metadata names are unique unless the type is qualified as \code{PS_META_NON_UNIQUE}; +in this case a unique integer will be added to each name that you specify. In this case, +you may either delete individual element separately or as a complete set: +\begin{verbatim} +psMetaDataItemFree(psMetaDataRemove(ms, "lang.hello.0")); +psMetaDataItemFree(psMetaDataRemove(ms, "lang.hello")); +\end{verbatim} + +\appendix +\section{MetaData APIs} + +\begin{verbatim} +psMetaDataItem *psMetaDataItemAlloc( + psMetaDataType type, // type of this piece of metadata + const void *val, // value of new item + // N.b. a pointer even if the item + // is of type e.g. int + const char *comment, // comment associated with item + const char *name, // name of new item of metadata (may be an sprintf format) + ...); // possible arguments for name format + +void psMetaDataItemFree(psMetaDataItem *ms); // piece of metadata to destroy + +psMetaDataSet *psMetaDataSetAlloc(void); // make a new set of metadata +void psMetaDataSetDel(psMetaDataSet *ms); // destroy a set of metadata + +/*****************************************************************************/ +/* + * Utilities + */ +psMetaDataItem *psMetaDataAppend(psMetaDataSet *restrict ms, psMetaDataItem *restrict item); +psMetaDataItem *psMetaDataRemove(psMetaDataSet *restrict ms, const char *restrict key); + +void psMetaDataSetIterator(psMetaDataSet *ms); +psMetaDataItem *psMetaDataGetNext(psMetaDataSet *ms); +psMetaDataItem *psMetaDataLookup(const psMetaDataSet *ms, const char *key); + +void psMetaDataItemPrint(FILE *fd, // file descriptor to write to + const psMetaDataItem *ms); // item of metadata to print +\end{verbatim} \bibliographystyle{plain} Index: /trunk/doc/draft/utils.tex =================================================================== --- /trunk/doc/draft/utils.tex (revision 178) +++ /trunk/doc/draft/utils.tex (revision 179) @@ -58,10 +58,10 @@ The \PS{} software system will need a level of memory management placed between the operating system (malloc/free) and the high level -routines (e.g. \code{psMetaDataNew}). +routines (e.g. \code{psMetaDataAlloc}). This layer is in addition to the possibility that specific heavily used data types may need their own special-purpose memory managers; but as we have specified that all user-level objects be allocated via -\code{typeNew/typeDel} functions, we will easily be able to +\code{typeAlloc/typeFree} functions, we will easily be able to implement such functionality without impacting on the facilities described here. @@ -69,7 +69,4 @@ All of the memory management APIs should be provided in a header file \file{psMemory.h} which is included by \file{psUtils.h}. - -\textit{Do we want to rename \code{psAlloc} to \code{psNew} and - \code{psFree} to \code{psDel}? Or is it too late? XXX} \subsection{Rationale} @@ -92,5 +89,5 @@ We need to provide a mechanism for tracking and fixing memory leaks. While it is possible to do this by linking with external - libraries (XXX reference), it is convenient to do so within the + libraries (\tbd{reference}), it is convenient to do so within the \PS{} framework. @@ -376,5 +373,5 @@ } psSimple; -psSimple *psSimpleNew(const char *name, int val) +psSimple *psSimpleAlloc(const char *name, int val) { psSimple *simp = psAlloc(sizeof(psSimple)); @@ -385,5 +382,5 @@ } -void psSimpleDel(psSimple *simp) +void psSimpleFree(psSimple *simp) { if (simp == NULL) { return; } @@ -411,8 +408,8 @@ \goodbreak \begin{verbatim} -simp = psSimpleNew("RHL", 0); +simp = psSimpleAlloc("RHL", 0); psDlistAppend(list1, psMemIncrRefCounter(simp)); psDlistAppend(list2, psMemIncrRefCounter(simp)); -psSimpleDel(simp); +psSimpleFree(simp); \end{verbatim} @@ -447,4 +444,6 @@ int level); // desired trace level int psGetTraceLevel(const char *name); // facilty of interest + +void psTraceReset(void); // turn off all tracing, and free trace's allocated memory void psPrintTraceLevels(void); // print trace levels @@ -513,4 +512,9 @@ within the functions that implement \file{psTrace.h} or \file{psMemory.h} systems. + + \item + \code{psTraceReset} shall reset all tracing to the state that it + had at program initiation, including freeing any memory that the + tracing subsystem may have allocated. \end{itemize} @@ -581,4 +585,6 @@ int psSetLogDestination(int dest); +void psSetLogFormat(const char *fmt); + int psSetLogLevel(int level); \end{verbatim} @@ -634,5 +640,9 @@ write to \code{stderr} and \code{stdout} respectively. -\section{The Pan-STARRS \texttt{psDlist} doubly-linked list type} +The fields included in the log message may be controlled using \code{psSetLogFormat} which +expects a string consisting of the letters \code{H} (host), \code{L} (level), \code{M} (message), +\code{N} (name), and \code{T} (time). The default is \code{HLMNT}. + +\section{The Pan-STARRS \code{psDlist} doubly-linked list type} \hlabel{psDlist} @@ -667,8 +677,8 @@ \begin{verbatim} -psDlist *psDlistNew(void *data); // initial data item; may be NULL - -void psDlistDel(psDlist *list, // list to destroy - void (*elemDel)(void *)); // destructor for list data, or NULL +psDlist *psDlistAlloc(void *data); // initial data item; may be NULL + +void psDlistFree(psDlist *list, // list to destroy + void (*elemFree)(void *)); // destructor for list data, or NULL /* @@ -702,14 +712,18 @@ their reference counter. -If \code{psDlistDel}'s argument \code{elemDel} is NULL, the +If \code{psDlistFree}'s argument \code{elemFree} is NULL, the list should be deleted, but not the elements on it (although their \code{refcounter}'s should be decremented). -The \code{psDlistGet} routine may be used to provide an iterator. The -call \code{psDlistGet(list, psDlistHead)} returns the data from the -head link of the list, but also sets the iteration cursor to the head. -A seried of calls to \code{psDlistGet(list, psDlistNext)} return the -elements of the list in order (you may also use \code{psDlistTail} and -\code{psDlistPrev} to traverse the list backwards). +Iteration over all elements of the list is provided by the functions: +\begin{verbatim} +void psDlistSetIterator(psDlist *list, int where, int which); +void *psDlistGetNext(psDlist *list, int which); +void *psDlistGetPrev(psDlist *list, int which); +\end{verbatim} +in which the \code{where} argument must be \code{PS_DLIST_HEAD} or \code{PS_DLIST_TAIL}, +to start at the head or tail of the list, and \code{psDlistGetNext} and \code{psDlistGetPrev} +return the next/previous element. The argument \code{which} identifies which of potentially +many iteration cursors should be used; it must currently always be \code{0}. Explicit traversal of the list using the \code{psDlistElem}'s @@ -746,7 +760,7 @@ with associated constructors and a destructor: \begin{verbatim} -psTypeArray *psTypeNew(int n, int size); +psTypeArray *psTypeAlloc(int n, int size); psTypeArray *psTypeRealloc(psTypeArray *arr, int n); -void psTypeDel(psTypeArray *arr); +void psTypeFree(psTypeArray *arr); \end{verbatim} @@ -760,5 +774,5 @@ and declares the prototypes (and is thus suitable for use in a header file); the latter generates the code for the three functions -\code{psType(New|Realloc|Del} (and should thus appear in exactly one +\code{psType(Alloc|Realloc|Free} (and should thus appear in exactly one source file for a given type). @@ -772,6 +786,6 @@ contains an array of \code{psType}s not pointers to \code{psType}s; this means that the individual elements are -not allocated using \code{psTypeNew}, are not correctly initialized, -and shouldn't be individually deleted with \code{psTypeDel}; +not allocated using \code{psTypeAlloc}, are not correctly initialized, +and shouldn't be individually deleted with \code{psTypeFree}; If you wish to use arrays of pointers, use the macros @@ -788,15 +802,15 @@ with associated constructors and a destructor: \begin{verbatim} -psTypePtrArray *psTypePtrNew(int n, int size); +psTypePtrArray *psTypePtrAlloc(int n, int size); psTypePtrArray *psTypePtrRealloc(psTypePtrArray *arr, int n); -void psTypePtrArrayDel(psTypePtrArray *arr); +void psTypePtrArrayFree(psTypePtrArray *arr); \end{verbatim} These constructors create arrays of \code{psType *} and call -\code{psTypeNew} and \code{psTypeDel} to allocate and free the +\code{psTypeAlloc} and \code{psTypeFree} to allocate and free the elements. As for the simple arrays, The former defines the typedef and declares the prototypes (and is thus suitable for use in a header file) and the latter generates the code for the three functions -\code{psType(New|Realloc|Del} (and should thus appear in exactly one +\code{psType(Alloc|Realloc|Free} (and should thus appear in exactly one source file for a given type). @@ -805,10 +819,10 @@ need to say something like: \begin{verbatim} - psTypePtrArray *pt = psTypePtrArrayNew(10, 10); + psTypePtrArray *pt = psTypePtrArrayAlloc(10, 10); psType *xy = psMemDecrRefCounter(pt->arr[0]); pt->arr[0] = NULL; \end{verbatim} -\subsubsection{Arrays of \texttt{void *}} +\subsubsection{Arrays of \code{void *}} \hlabel{secArrayVoidPtr} @@ -825,14 +839,14 @@ except that its destructor is specified as: \begin{verbatim} -void psVoidPtrArrayDel(psVoidPtrArray *arr, // array to destroy - void (*elemDel)(void *)); // destructor for array data -\end{verbatim} - -The routine \code{psVoidPtrArrayDel} assumes that all pointers +void psVoidPtrArrayFree(psVoidPtrArray *arr, // array to destroy + void (*elemFree)(void *)); // destructor for array data +\end{verbatim} + +The routine \code{psVoidPtrArrayFree} assumes that all pointers had their reference counters incremented when they were inserted onto the array.\footnote{% \eg{} \code{va->arr[i] = psMemIncrRefCounter(ptr);}} -If \code{psVoidPtrArrayDel}'s argument \code{elemDel} is NULL, the +If \code{psVoidPtrArrayFree}'s argument \code{elemFree} is NULL, the list should be deleted, but not the elements on it (although their \code{refcounter}'s should be decremented). @@ -849,10 +863,10 @@ } psXY; -psXY *psXYNew(void) +psXY *psXYAlloc(void) { return psAlloc(sizeof(psXY)); } -void psXYDel(psXY *xy) +void psXYFree(psXY *xy) { psFree(xy); @@ -867,6 +881,6 @@ int main(void) { - psXYArray *t = psXYArrayNew(10, 15); - psXYPtrArray *pt = psXYPtrArrayNew(10, 10); + psXYArray *t = psXYArrayAlloc(10, 15); + psXYPtrArray *pt = psXYPtrArrayAlloc(10, 10); for (int i = 0; i < t->n; i++) { @@ -883,11 +897,11 @@ printf("\n"); - psXYArrayDel(t); + psXYArrayFree(t); psXY *xy = psMemDecrRefCounter(pt->arr[0]); pt->arr[0] = NULL; - psXYDel(xy); + psXYFree(xy); - psXYPtrArrayDel(pt); + psXYPtrArrayFree(pt); psMemCheckLeaks(0, NULL, stderr); @@ -904,16 +918,19 @@ typedef struct HashTable psHash; -psHash *psHashNew(int nbucket); // initial number of buckets -void psHashDel(psHash *table, // hash table to be freed - void (*itemDel)(void *item)); // how to free hashed data; - // or NULL - -void *psHashInsert(psHash *table, // table to insert in - const char *key, // key to use - void *data, // data to insert - void (*itemDel)(void *item)); // how to free hashed data; - // or NULL -void *psHashLookup(psHash *table, // table to lookup key in - const char *key); // key to lookup +psHash *psHashAlloc(int nbucket); // initial number of buckets +void psHashFree(psHash *table, // hash table to be freed + void (*itemFree)(void *item)); // how to free hashed data; + // or NULL + +void *psHashInsert(psHash *table, // table to insert in + const char *key, // key to use + void *data, // data to insert + void (*itemFree)(void *item)); // how to free hashed data; + // or NULL +void *psHashLookup(psHash *table, // table to lookup key in + const char *key); // key to lookup + +void *psHashRemove(psHash *table, // table to lookup key in + const char *key); // key to lookup \end{verbatim} \begin{caption}{The public API for hash tables from \file{psHash.h}} @@ -926,25 +943,28 @@ is included by \file{psUtils.h}. \footnote{ - We choose not to use the posix function \texttt{hcreate}, - \texttt{hdestroy}, and \code{hsearch} as they only support + We choose not to use the posix function \code{hcreate}, + \code{hdestroy}, and \code{hsearch} as they only support a single hash table at any one time.} -A hash table is an abstract type \texttt{psHash}. The argument -\texttt{nbucket} to \texttt{psHashNew} is a non-binding suggestion +A hash table is an abstract type \code{psHash}. The argument +\code{nbucket} to \code{psHashAlloc} is a non-binding suggestion from the user for the initial size of the hash table. -If the \texttt{itemDel} argument to \texttt{psHashDel} is non-NULL, +If the \code{itemFree} argument to \code{psHashFree} is non-NULL, it will be used to delete the data items that have been stored in the hash table; if it is NULL this is the responsibility of the caller. -The routine \texttt{psHashInsert} must provide a non-NULL \texttt{itemDel} +The routine \code{psHashInsert} must provide a non-NULL \code{itemFree} argument if it wishes to change the value previously inserted keys; -if \texttt{itemDel} is NULL attempting to insert a pre-existing key -is an error, and the routine will return NULL. If \texttt{psHashInsert} -succeeds it returns \texttt{data}. - -\texttt{psHashLookup} returns the \texttt{data} associated with the +if \code{itemFree} is NULL attempting to insert a pre-existing key +is an error, and the routine will return NULL. If \code{psHashInsert} +succeeds it returns \code{data}. + +\code{psHashLookup} returns the \code{data} associated with the key, or NULL if the key's invalid. + +\code{psHashRemove} removes the entry associated with the +key from the table, and returns the \code{data}; if the key's invalid it returns NULL. \section{Miscellaneous Utilities}