Changeset 31568

Timestamp:

May 25, 2011, 2:16:18 PM (15 years ago)

Author:

eugene

Message:

adding element to psMemBlock to allow more fine-grained locking

Location:

branches/eam_branches/ipp-20110505/psLib/src/sys

Files:

• psMemory.c (modified) (4 diffs)
• psMemory.h (modified) (1 diff)

branches/eam_branches/ipp-20110505/psLib/src/sys/psMemory.c

@@ -328 +328 @@
 // This is the root of the entire memory list
 static psMemBlock *lastMemBlockAllocated = NULL;
+static bool lastMemBlockInFlight = false;
 
 /* Actually allocate memory
@@ -536 +537 @@
     // Reallocate the memory
 
-    // we need to lock this whole section because we need to be able to adjust
-    // lastMemBlockAllocated if needed (ie, this IS lastMemBlockAllocated)
-    // this lock is frequently called in a given program
+    /* psRealloc and locks: 
+
+       psRealloc calls 'realloc' to change the size of the memory block.  This call is likely
+       to change the address of the memory block itself.  This is a problem because the address
+       of the memory block is referred to by the neighbor blocks via their memBlock->nextBlock
+       and memBlock->previousBlock elements.  If the current memBlock is the first one, then
+       this issue also applies to 'lastMemBlockAllocated'.  The elements may also be modified
+       by psFree and psAlloc.  Thus, we need to prevent multiple threads from calling psFree,
+       psAlloc, or psRealloc on neighbor memory blocks at the same time.
+
+       Unfortunately, unlike psAlloc and psFree, psRealloc cannot perform the (expensive)
+       system call operation (realloc) first and adjust the pointers in separate step (since
+       the value modified by realloc *is* one of those points.  It is thus necessary to include
+       the realloc call within the locked segement, making psRealloc likely to serialize the
+       thread operations.
+
+       Alternatively, we can recognize that the lock only need be applied to operations which
+       are performed on neighboring memBlocks.  We can thus reduce the contention by having a
+       per-memBlock boolean (inFlight) which says the memBlock or a neighbor is being
+       modified.  If psFree and psAlloc respect that boolean, they will not modify a memBlock
+       which is already being realloced (or its neighbor).  In this way, the 'realloc' call can
+       be performed outside of the locked region.
+
+    */
+
+    // set a lock, then lock this memblock and its neighbors
     MUTEX_LOCK(&memBlockListMutex);
 
@@ -548 +572 @@
     // lastMemBlockAllocated will be left with a bogus value
     bool isBlockLast = (memBlock == lastMemBlockAllocated);
+
+    memBlock->inFlight = true;
+    nextBlock->inFlight = true;
+    previousBlock->inFlight = true;
+    if (isBlockLast) lastMemBlockInFlight = true;
+
+    MUTEX_UNLOCK(&memBlockListMutex);
+
+    // now that the per-memBlock locks are set, we can unlock the threads and realloc the data
 
     memBlock = (psMemBlock *)realloc(memBlock, sizeof(psMemBlock) + size + sizeof(void *));
@@ -581 +614 @@
         memAllocID += memAllocCallback(memBlock);
     }
+
+    // all of the list modifications are done; set the lock and clear the per-memBlock locks
+    MUTEX_LOCK(&memBlockListMutex);
+    memBlock->inFlight = false;
+    nextBlock->inFlight = false;
+    previousBlock->inFlight = false;
+    if (isBlockLast) lastMemBlockInFlight = false;
 
     MUTEX_UNLOCK(&memBlockListMutex);

branches/eam_branches/ipp-20110505/psLib/src/sys/psMemory.h

@@ -73 +73 @@
 
     psReferenceCount refCounter;        ///< how many times pointer is referenced
-    bool persistent;                    ///< marks if this non-user persistent data like error stack, etc.
+    bool persistent;                    ///< true if this is non-user persistent data like error stack, etc.
+    bool inFlight;                      ///< true if a nearby block is being free'ed / realloc'ed (local lock)
     const uint32_t endblock;            ///< initialised to p_psMEMMAGIC
 }