Index: /branches/eam_branches/ipp-20110505/psLib/src/sys/psMemory.c
===================================================================
--- /branches/eam_branches/ipp-20110505/psLib/src/sys/psMemory.c	(revision 31574)
+++ /branches/eam_branches/ipp-20110505/psLib/src/sys/psMemory.c	(revision 31575)
@@ -332,5 +332,65 @@
 
 // if HARDLOCKS is set, revert to global locking for realloc
-# define HARDLOCKS 1
+# define HARDLOCKS 0
+
+# if (!HARDLOCKS)
+static int setLock1 = 0;
+static int clearLock1 = 0;
+
+static int setLock2 = 0;
+static int clearLock2 = 0;
+
+static int setLockNext = 0;
+static int clearLockNext = 0;
+
+static int setLockPrev = 0;
+static int clearLockPrev = 0;
+# endif
+
+# define BLOCK_SLEEP 100
+
+static int callL1 = 0;
+static int callL2 = 0;
+
+# define MEM_ASSERT(COND,MSG) { if (!(COND)) { fprintf (stderr, MSG); abort(); } }
+
+// pointer to the last mem block that was allocated.
+// This is the root of the entire memory list
+static psMemBlock *lastMemBlockAllocated = NULL;
+
+// set lock on lastMemBlockAllocated
+void BLOCKLAST_LOCK () {
+
+# if (HARDLOCKS) 
+    MUTEX_LOCK(&memBlockListMutex);
+# else
+
+    // if memBlock is not defined, we are just asking for a m
+    if (!lastMemBlockAllocated) {
+	MUTEX_LOCK(&memBlockListMutex);
+	return;
+    }
+	
+    while (true) {
+	// we cannot set the lock on the marker while the lock is held
+	while (lastMemBlockAllocated->inFlight) { usleep (BLOCK_SLEEP); }
+
+	// set a lock, then lock this memblock and its neighbors
+	MUTEX_LOCK(&memBlockListMutex);
+    
+	// did we beat the race condition?  is the lock still clear?
+	if (lastMemBlockAllocated->inFlight) {
+	    // nope, we need to try again
+	    MUTEX_UNLOCK(&memBlockListMutex);
+	    continue;
+	}
+
+	// the marker is ours!
+	lastMemBlockAllocated->inFlight = true;
+	setLock1 ++;
+	return;
+    }
+# endif
+}
 
 // memBlock->inFlight is a 'soft' lock.  if true, the lock is set
@@ -341,5 +401,5 @@
 # else
 
-    psAssert (memBlock || keepMutex, "trying to set a soft lock on a non-existent memBlock");
+    MEM_ASSERT (memBlock || keepMutex, "trying to set a soft lock on a non-existent memBlock\n");
 
     // if memBlock is not defined, we are just asking for a m
@@ -351,5 +411,5 @@
     while (true) {
 	// we cannot set the lock on the marker while the lock is held
-	while (memBlock->inFlight) { usleep (10000); }
+	while (memBlock->inFlight) { usleep (BLOCK_SLEEP); }
 
 	// set a lock, then lock this memblock and its neighbors
@@ -365,4 +425,5 @@
 	// the marker is ours!
 	memBlock->inFlight = true;
+	setLock1 ++;
 	if (keepMutex) return;
 
@@ -379,5 +440,5 @@
 # else
 
-    psAssert (memBlock || haveMutex, "trying to clear a soft lock on a non-existent memBlock");
+    MEM_ASSERT (memBlock || haveMutex, "trying to clear a soft lock on a non-existent memBlock\n");
 
     // if memBlock is not defined, we are just asking for a regular lock
@@ -387,4 +448,6 @@
     }
 
+    MEM_ASSERT (memBlock->inFlight, "trying to clear an unlocked memBlock\n");
+
     // need to lock before clearing the marker
     if (!haveMutex) {
@@ -392,4 +455,5 @@
     }
     memBlock->inFlight = false;
+    clearLock1 ++;
     MUTEX_UNLOCK(&memBlockListMutex);
     return;
@@ -406,16 +470,12 @@
 # else
 
-    psAssert (memBlock, "trying to set a soft lock on a non-existent memBlock");
+    MEM_ASSERT (memBlock, "trying to set a soft lock on a non-existent memBlock\n");
 
     while (true) {
 	// we cannot set the lock on the marker while the lock is held
 	// wait until all three markers are clear:
-	while (memBlock->inFlight) { usleep (10000); }
-	if (memBlock->nextBlock) {
-	    while (memBlock->nextBlock->inFlight) { usleep (10000); }
-	}
-	if (memBlock->previousBlock) {
-	    while (memBlock->previousBlock->inFlight) { usleep (10000); }
-	}
+	while (memBlock->inFlight) { usleep (BLOCK_SLEEP); }
+	while (memBlock->nextBlock && memBlock->nextBlock->inFlight) { usleep (BLOCK_SLEEP); }
+	while (memBlock->previousBlock && memBlock->previousBlock->inFlight) { usleep (BLOCK_SLEEP); }
 
 	// set a lock, then lock this memblock and its neighbors
@@ -441,9 +501,12 @@
 	// the markers are ours!
 	memBlock->inFlight = true;
+	setLock2 ++;
 	if (memBlock->nextBlock) {
 	    memBlock->nextBlock->inFlight = true;
+	    setLockNext ++;
 	}
 	if (memBlock->previousBlock) {
 	    memBlock->previousBlock->inFlight = true;
+	    setLockPrev ++;
 	}
 	if (keepMutex) return;
@@ -465,10 +528,16 @@
 	MUTEX_LOCK(&memBlockListMutex);
     }
+    MEM_ASSERT (memBlock->inFlight, "trying to clear an unlocked memBlock\n");
     memBlock->inFlight = false;
+    clearLock2 ++;
     if (memBlock->nextBlock) {
+	MEM_ASSERT (memBlock->nextBlock->inFlight, "trying to clear an unlocked memBlock\n");
 	memBlock->nextBlock->inFlight = false;
+	clearLockNext ++;
     }
     if (memBlock->previousBlock) {
+	MEM_ASSERT (memBlock->previousBlock->inFlight, "trying to clear an unlocked memBlock\n");
 	memBlock->previousBlock->inFlight = false;
+	clearLockPrev ++;
     }
     MUTEX_UNLOCK(&memBlockListMutex);
@@ -476,8 +545,4 @@
 # endif
 }
-
-// pointer to the last mem block that was allocated.
-// This is the root of the entire memory list
-static psMemBlock *lastMemBlockAllocated = NULL;
 
 /* Actually allocate memory
@@ -549,8 +614,19 @@
     memBlock->refCounter = 1;                   // one user so far
 
+    psMemBlock *last0 = lastMemBlockAllocated;
+    int flight0 = (lastMemBlockAllocated) ? lastMemBlockAllocated->inFlight : 10;
+
     // need exclusive access of the memory block list now...
     // this lock is very frequently called in a given program
     // lastMemBlockAllocated is always true except the first allocation
-    BLOCK_LOCK (lastMemBlockAllocated, true);
+    BLOCKLAST_LOCK ();
+    callL1++;
+
+    psMemBlock *last1 = lastMemBlockAllocated;
+    int flight1 = (lastMemBlockAllocated) ? lastMemBlockAllocated->inFlight : 10;
+
+    if (false) {
+	fprintf (stderr, "last 0 : %lld, last 1 : %lld, flight0: %d, flight1: %d\n", (long long int) last0, (long long int) last1, (int) flight0, (int) flight1);
+    }
 
     // increment the memory id only after we've grabbed the memBlockListMutex this value is
@@ -573,4 +649,8 @@
 
     BLOCK_UNLOCK (memBlock->nextBlock, true);
+    callL1--;
+
+    // if (callL1 > +7) abort();
+    // if (callL1 < -7) abort();
 
     // And return the user the memory that they allocated
@@ -720,6 +800,6 @@
     // set a lock, then lock this memblock and its neighbors.  at the end of this call,
     // the global mutex is released, but the memBlock and its neighbors are protected
-    BLOCKSET_LOCK (memBlock, false);
-    // MUTEX_LOCK(&memBlockListMutex);
+    // BLOCKSET_LOCK (memBlock, false);
+    MUTEX_LOCK(&memBlockListMutex);
 
     psMemBlock *nextBlock = memBlock->nextBlock;
@@ -767,10 +847,11 @@
 
     // all of the list modifications are done; set the lock and clear the per-memBlock locks
-    // MUTEX_UNLOCK(&memBlockListMutex);
-    BLOCKSET_UNLOCK(memBlock, false);
-
-    psAssert (!memBlock->inFlight, "unreleased lock?");
-    psAssert (!nextBlock || !nextBlock->inFlight, "unreleased lock?");
-    psAssert (!previousBlock || !previousBlock->inFlight, "unreleased lock?");
+    // BLOCKSET_UNLOCK(memBlock, false);
+    MUTEX_UNLOCK(&memBlockListMutex);
+
+    // XXX these are not actually guaranteed : another thread may already grab them before we get here
+    // psAssert (!memBlock->inFlight, "unreleased lock?");
+    // psAssert (!nextBlock || !nextBlock->inFlight, "unreleased lock?");
+    // psAssert (!previousBlock || !previousBlock->inFlight, "unreleased lock?");
 
     return memBlock + 1;                    // usr memory
@@ -999,4 +1080,5 @@
     // we need a MUTEX_LOCK here: otherwise, two functions can race on refCounter--;
     BLOCK_LOCK (memBlock, true);
+    callL2++;
     if (memBlock->refCounter > 1) {
         memBlock->refCounter--;
@@ -1007,7 +1089,11 @@
         }
 	BLOCK_UNLOCK(memBlock, true);
+	callL2--;
         return ptr;
     }
     BLOCK_UNLOCK(memBlock, true);
+    callL2--;
+    // if (callL2 > +7) abort();
+    // if (callL2 < -7) abort();
 
     // we can't invoke freeFunc() while we're holding memBlockListMutex as it
@@ -1041,12 +1127,16 @@
     BLOCKSET_UNLOCK (memBlock, true);
 
+    // XXX keep this?
+    psAssert (memBlock->nextBlock == nextBlock, "unexpected rearrangement");
+    psAssert (memBlock->previousBlock == previousBlock, "unexpected rearrangement");
+
     // NULL out the refs so no one can get confused
     memBlock->nextBlock = NULL;
     memBlock->previousBlock = NULL;
 
-    // XXX these are slightly dangerous : someone else may grab the lock in the meanwhile...
-    psAssert (!memBlock->inFlight, "unreleased lock?");
-    psAssert (!nextBlock || !nextBlock->inFlight, "unreleased lock?");
-    psAssert (!previousBlock || !previousBlock->inFlight, "unreleased lock?");
+    // XXX these are not actually guaranteed : another thread may already grab them before we get here
+    // psAssert (!memBlock->inFlight, "unreleased lock?");
+    // psAssert (!nextBlock || !nextBlock->inFlight, "unreleased lock?");
+    // psAssert (!previousBlock || !previousBlock->inFlight, "unreleased lock?");
 
     // invoke free only after we've released the block list lock as free()