Update CUtlMemoryPoolBase after 4/2/2024 update (#228)

Also updates tier0/tslist.h to sdk2013 variant and other minor stuff.

public/inetchannel.h

@@ -127,7 +127,7 @@ public:
 	
 	virtual void	PostReceivedNetMessage( INetworkSerializable *pNetMessage, const void *pData, const NetChannelBufType_t *pBufType, int nBits, int nInSequenceNr ) = 0;
 	virtual void	InsertReplayMessage( InstantReplayMessage_t &msg ) = 0;
-	virtual bool	HasQueuedPackets( int nMessageId ) const = 0;
+	virtual bool	HasQueuedNetMessages( int nMessageId ) const = 0;
 
 	virtual void	SetPendingDisconnect( ENetworkDisconnectionReason reason ) = 0;
 	virtual ENetworkDisconnectionReason GetPendingDisconnect( void ) const = 0;
@@ -135,10 +135,10 @@ public:
 	virtual void	SuppressTransmit( bool suppress ) = 0;
 	virtual bool	IsSuppressingTransmit( void ) const = 0;
 	
-	virtual EResult	SendMessage( const void *pData, uint32 cbData, int nSendFlags ) = 0;
+	virtual EResult	SendRawMessage( const void *pData, uint32 cbData, int nSendFlags ) = 0;
 	
-	virtual int		GetCurrentMessageBits( void ) const = 0;
+	virtual int		GetCurrentNetMessageBits( void ) const = 0;
-	virtual int		GetCurrentMessageInSequenceNr( void ) const = 0;
+	virtual int		GetCurrentNetMessageInSequenceNr( void ) const = 0;
 
 	virtual void	unk101( void ) = 0;
 	virtual void	unk102( void ) = 0;

public/schemasystem/schemasystem.h

@@ -76,8 +76,8 @@ public:
 	virtual void MarkEnumAsRequiringGlobalPromotion( const CSchemaEnumInfo* pEnumInfo ) = 0;
 
 	virtual void ResolveAtomicInfoThreadsafe( const SchemaAtomicTypeInfo_t** ppAtomicInfo, const char* pszAtomicName, int nAtomicID ) = 0;
-	virtual void ResolveEnumInfoThreadsafe( const CSchemaEnumInfo** pEnumInfo, const char* pszEnumName ) = 0;
+	virtual void ResolveEnumInfoThreadsafe( const CSchemaEnumInfo** ppEnumInfo, const char* pszEnumName ) = 0;
-	virtual void ResolveClassInfoThreadsafe( const CSchemaClassInfo** pClassInfo, const char* pszClassName ) = 0;
+	virtual void ResolveClassInfoThreadsafe( const CSchemaClassInfo** ppClassInfo, const char* pszClassName ) = 0;
 };
 
 class CSchemaSystemTypeScope : public ISchemaSystemTypeScope

public/tier0/threadtools.h

@@ -289,7 +289,9 @@ inline bool ThreadInterlockedAssignIf128( volatile int128 *pDest, const int128 &
 	int64 *pValue64 = ( int64 * )&value;
 	int64 *pComperand64 = ( int64 * )&comperand;
 	
-	return _InterlockedCompareExchange128( pDest64, pValue64[1], pValue64[0], pComperand64 ) == 1;
+	int64 local_comperand[2] = { pComperand64[0], pComperand64[1] };
+
+	return _InterlockedCompareExchange128( pDest64, pValue64[1], pValue64[0], local_comperand ) == 1;
 }
 #else
 typedef __int128_t int128;
@@ -298,7 +300,10 @@ typedef __int128_t int128;
 inline bool ThreadInterlockedAssignIf128( volatile int128 *pDest, const int128 &value, const int128 &comperand )
 {
 	Assert( (size_t)pDest % 16 == 0 );
-	return __sync_bool_compare_and_swap( pDest, comperand, value );
+	
+	int128 local_comperand = comperand;
+	
+	return __sync_bool_compare_and_swap( pDest, local_comperand, value );
 }
 #endif
 #endif

public/tier0/tslist.h

@@ -1,4 +1,4 @@
-//========== Copyright <20> 2005, Valve Corporation, All rights reserved. ========
+//========= Copyright Valve Corporation, All rights reserved. ============//
 //
 // Purpose:
 //
@@ -12,10 +12,10 @@
 
 #if defined( _WIN32 )
 #pragma once
-#endif
+// Suppress this spurious warning:
-
+// warning C4700: uninitialized local variable 'oldHead' used
-#if ( defined( PLATFORM_X360 ) || defined( PLATFORM_WINDOWS_PC64 ) )
+#pragma warning( push )
-#define USE_NATIVE_SLIST
+#pragma warning( disable : 4700 )
 #endif
 
 #if defined( USE_NATIVE_SLIST ) && !defined( _X360 )
@@ -25,21 +25,50 @@
 
 #include "tier0/dbg.h"
 #include "tier0/threadtools.h"
+#include "tier0/memalloc.h"
+#include "tier0/memdbgoff.h"
 
-#include "tier0/memdbgon.h"
+#if defined( _X360 )
+#define USE_NATIVE_SLIST
+#endif
 
 //-----------------------------------------------------------------------------
 
-#if defined(PLATFORM_WINDOWS_PC64)
+#if defined( PLATFORM_64BITS )
-#define TSLIST_HEAD_ALIGNMENT 16 //MEMORY_ALLOCATION_ALIGNMENT
+
-#define TSLIST_NODE_ALIGNMENT 16 //MEMORY_ALLOCATION_ALIGNMENT
+#define TSLIST_HEAD_ALIGNMENT 16
+#define TSLIST_NODE_ALIGNMENT 16
+inline bool ThreadInterlockedAssignIf64x128( volatile int128 *pDest, const int128 &value, const int128 &comperand )
+	{ return ThreadInterlockedAssignIf128( pDest, value, comperand ); }
 #else
 #define TSLIST_HEAD_ALIGNMENT 8
 #define TSLIST_NODE_ALIGNMENT 8
+inline bool ThreadInterlockedAssignIf64x128( volatile int64 *pDest, const int64 value, const int64 comperand )
+	{ return ThreadInterlockedAssignIf64( pDest, value, comperand ); }
 #endif
 
+#ifdef _MSC_VER
 #define TSLIST_HEAD_ALIGN DECL_ALIGN(TSLIST_HEAD_ALIGNMENT)
 #define TSLIST_NODE_ALIGN DECL_ALIGN(TSLIST_NODE_ALIGNMENT)
+#define TSLIST_HEAD_ALIGN_POST
+#define TSLIST_NODE_ALIGN_POST
+#elif defined( GNUC )
+#define TSLIST_HEAD_ALIGN 
+#define TSLIST_NODE_ALIGN 
+#define TSLIST_HEAD_ALIGN_POST DECL_ALIGN(TSLIST_HEAD_ALIGNMENT)
+#define TSLIST_NODE_ALIGN_POST DECL_ALIGN(TSLIST_NODE_ALIGNMENT)
+#elif defined( _PS3 )
+#define TSLIST_HEAD_ALIGNMENT 8
+#define TSLIST_NODE_ALIGNMENT 8
+
+#define TSLIST_HEAD_ALIGN ALIGN8
+#define TSLIST_NODE_ALIGN ALIGN8
+#define TSLIST_HEAD_ALIGN_POST ALIGN8_POST
+#define TSLIST_NODE_ALIGN_POST ALIGN8_POST
+
+#else
+#error
+#endif
 
 //-----------------------------------------------------------------------------
 
@@ -58,38 +87,94 @@ typedef SLIST_HEADER TSLHead_t;
 struct TSLIST_NODE_ALIGN TSLNodeBase_t
 {
 	TSLNodeBase_t *Next; // name to match Windows
-};
+} TSLIST_NODE_ALIGN_POST;
 
-union TSLHead_t
+union TSLIST_HEAD_ALIGN TSLHead_t
 {
 	struct Value_t
 	{
 		TSLNodeBase_t *Next;
+		// <sergiy> Depth must be in the least significant halfword when atomically loading into register,
+		//          to avoid carrying digits from Sequence. Carrying digits from Depth to Sequence is ok,
+		//          because Sequence can be pretty much random. We could operate on both of them separately,
+		//          but it could perhaps (?) lead to problems with store forwarding. I don't know 'cause I didn't 
+		//          performance-test or design original code, I'm just making it work on PowerPC.
+		#ifdef VALVE_BIG_ENDIAN
+		int16	Sequence;
+		int16   Depth;
+		#else
 		int16   Depth;
 		int16	Sequence;
+		#endif
+#ifdef PLATFORM_64BITS
+		int32   Padding;
+#endif
 	} value;
 
-	int64 value64;
+	struct Value32_t
-};
+	{
+		TSLNodeBase_t *Next_do_not_use_me;
+		int32   DepthAndSequence;
+	} value32;
+
+#ifdef PLATFORM_64BITS
+	int128 value64x128;
+#else
+	int64 value64x128;
+#endif
+} TSLIST_HEAD_ALIGN_POST;
+
 #endif
 
 //-------------------------------------
-
 class TSLIST_HEAD_ALIGN CTSListBase
 {
 public:
+
+	// override new/delete so we can guarantee 8-byte aligned allocs
+	static void * operator new( size_t size )
+	{
+		CTSListBase *pNode = (CTSListBase *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ );
+		return pNode;
+	}
+
+	static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine )
+	{
+		CTSListBase *pNode = (CTSListBase *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, pFileName, nLine );
+		return pNode;
+	}
+
+	static void operator delete( void *p)
+	{
+		MemAlloc_FreeAligned( p );
+	}
+
+	static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine )
+	{
+		MemAlloc_FreeAligned( p );
+	}
+
+private:
+	// These ain't gonna work
+	static void * operator new[] ( size_t size );
+	static void operator delete [] ( void *p);
+	
+public:
+
 	CTSListBase()
 	{
 		if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 )
 		{
-			Error( _T( "CTSListBase: Misaligned list\n" ) );
+			Plat_FatalErrorFunc( "CTSListBase: Misaligned list\n" );
 			DebuggerBreak();
 		}
 
 #ifdef USE_NATIVE_SLIST
 		InitializeSListHead( &m_Head );
+#elif defined(PLATFORM_64BITS)
+		m_Head.value64x128 = int128_zero();
 #else
-		m_Head.value64 = (int64)0;
+		m_Head.value64x128 = (int64)0;
 #endif
 	}
 
@@ -103,7 +188,7 @@ public:
 #ifdef _DEBUG
 		if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 )
 		{
-			Error( _T( "CTSListBase: Misaligned node\n" ) );
+			Plat_FatalErrorFunc( "CTSListBase: Misaligned node\n" );
 			DebuggerBreak();
 		}
 #endif
@@ -119,14 +204,23 @@ public:
 		TSLHead_t oldHead;
 		TSLHead_t newHead;
 
+		#if defined( PLATFORM_PS3 ) || defined( PLATFORM_X360 )
+		__lwsync(); // write-release barrier
+		#endif
+
+#ifdef PLATFORM_64BITS
+		newHead.value.Padding = 0;
+#endif
 		for (;;)
 		{
-			oldHead.value64 = m_Head.value64;
+			oldHead.value64x128 = m_Head.value64x128;
 			pNode->Next = oldHead.value.Next;
 			newHead.value.Next = pNode;
-			*((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) + 0x10001;
 			
-			if ( ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) )
+			newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence + 0x10001;
+			
+			
+			if ( ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) )
 			{
 				break;
 			}
@@ -151,17 +245,24 @@ public:
 		TSLHead_t oldHead;
 		TSLHead_t newHead;
 
+#ifdef PLATFORM_64BITS
+		newHead.value.Padding = 0;
+#endif
 		for (;;)
 		{
-			oldHead.value64 = m_Head.value64;
+			oldHead.value64x128 = m_Head.value64x128;
 			if ( !oldHead.value.Next )
 				return NULL;
 
 			newHead.value.Next = oldHead.value.Next->Next;
-			*((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) - 1;
+			newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence	- 1;
 
-			if ( ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) )
+
+			if ( ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) )
 			{
+				#if defined( PLATFORM_PS3 ) || defined( PLATFORM_X360 )
+					__lwsync(); // read-acquire barrier
+				#endif
 				break;
 			}
 			ThreadPause();
@@ -175,7 +276,7 @@ public:
 	{
 #ifdef USE_NATIVE_SLIST
 		TSLNodeBase_t *pBase = (TSLNodeBase_t *)InterlockedFlushSList( &m_Head );
-#ifdef _X360
+#if defined( _X360 ) || defined( _PS3 )
 		__lwsync(); // read-acquire barrier
 #endif
 		return pBase;
@@ -183,18 +284,24 @@ public:
 		TSLHead_t oldHead;
 		TSLHead_t newHead;
 
+#ifdef PLATFORM_64BITS
+		newHead.value.Padding = 0;
+#endif
 		do
 		{
 			ThreadPause();
 
-			oldHead.value64 = m_Head.value64;
+			oldHead.value64x128 = m_Head.value64x128;
 			if ( !oldHead.value.Next )
 				return NULL;
 
 			newHead.value.Next = NULL;
-			*((uint32 *)&newHead.value.Depth) = *((uint32 *)&oldHead.value.Depth) & 0xffff0000;
+			// <sergiy> the reason for AND'ing it instead of poking a short into memory 
+			//          is probably to avoid store forward issues, but I'm not sure because
+			//          I didn't construct this code. In any case, leaving it as is on big-endian
+			newHead.value32.DepthAndSequence = oldHead.value32.DepthAndSequence & 0xffff0000;
 
-		} while( !ThreadInterlockedAssignIf64( &m_Head.value64, newHead.value64, oldHead.value64 ) );
+		} while( !ThreadInterlockedAssignIf64x128( &m_Head.value64x128, newHead.value64x128, oldHead.value64x128 ) );
 
 		return (TSLNodeBase_t *)oldHead.value.Next;
 #endif
@@ -216,7 +323,7 @@ public:
 
 private:
 	TSLHead_t m_Head;
-};
+} TSLIST_HEAD_ALIGN_POST;
 
 //-------------------------------------
 
@@ -234,7 +341,7 @@ public:
 	{
 		return (T *)CTSListBase::Pop();
 	}
-};
+} TSLIST_HEAD_ALIGN_POST;
 
 //-------------------------------------
 // this is a replacement for CTSList<> and CObjectPool<> that does not
@@ -249,11 +356,16 @@ class TSLIST_HEAD_ALIGN CTSPool : public CTSListBase
 	struct TSLIST_NODE_ALIGN simpleTSPoolStruct_t : public TSLNodeBase_t
 	{
 		T elem;
-	};
+	} TSLIST_NODE_ALIGN_POST;
 
 public:
 
 	~CTSPool()
+	{
+		Purge();
+	}
+
+	void Purge()
 	{
 		simpleTSPoolStruct_t *pNode = NULL;
 		while ( 1 )
@@ -289,8 +401,7 @@ public:
 	{
 		return GetObject();
 	}
-
+} TSLIST_HEAD_ALIGN_POST;
-};
 //-------------------------------------
 
 template <typename T>
@@ -301,10 +412,32 @@ public:
 	{
 		Node_t() {}
 		Node_t( const T &init ) : elem( init ) {}
-
 		T elem;
 
-	};
+	    // override new/delete so we can guarantee 8-byte aligned allocs
+	    static void * operator new( size_t size )
+	    {
+      		Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, __FILE__, __LINE__ );
+			return pNode;
+	    }
+
+		// override new/delete so we can guarantee 8-byte aligned allocs
+		static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine )
+		{
+			Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, pFileName, nLine );
+			return pNode;
+		}
+
+	    static void operator delete( void *p)
+	    {
+			MemAlloc_FreeAligned( p );
+	    }
+		static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine )
+		{
+			MemAlloc_FreeAligned( p );
+		}
+
+	} TSLIST_NODE_ALIGN_POST;
 
 	~CTSList()
 	{
@@ -358,7 +491,7 @@ public:
 		return (Node_t *)CTSListBase::Detach();
 	}
 
-};
+} TSLIST_HEAD_ALIGN_POST;
 
 //-------------------------------------
 
@@ -372,7 +505,7 @@ public:
 		Node_t( const T &init ) : elem( init ) {}
 
 		T elem;
-	};
+	} TSLIST_NODE_ALIGN_POST;
 
 	~CTSListWithFreeList()
 	{
@@ -453,7 +586,7 @@ public:
 
 private:
 	CTSListBase m_FreeList;
-};
+} TSLIST_HEAD_ALIGN_POST;
 
 //-----------------------------------------------------------------------------
 // Lock free queue
@@ -473,37 +606,114 @@ template <typename T, bool bTestOptimizer = false>
 class TSLIST_HEAD_ALIGN CTSQueue
 {
 public:
+
+	// override new/delete so we can guarantee 8-byte aligned allocs
+	static void * operator new( size_t size )
+	{
+		CTSQueue *pNode = (CTSQueue *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ );
+		return pNode;
+	}
+
+	// override new/delete so we can guarantee 8-byte aligned allocs
+	static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine )
+	{
+		CTSQueue *pNode = (CTSQueue *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, pFileName, nLine );
+		return pNode;
+	}
+
+	static void operator delete( void *p)
+	{
+		MemAlloc_FreeAligned( p );
+	}
+
+	static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine )
+	{
+		MemAlloc_FreeAligned( p );
+	}
+
+private:
+	// These ain't gonna work
+	static void * operator new[] ( size_t size ) throw()
+	{
+		return NULL;
+	}
+
+	static void operator delete [] ( void *p)
+	{
+	}
+
+public:
+
 	struct TSLIST_NODE_ALIGN Node_t
 	{
+		// override new/delete so we can guarantee 8-byte aligned allocs
+		static void * operator new( size_t size )
+		{
+			Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, __FILE__, __LINE__ );
+			return pNode;
+		}
+
+		static void * operator new( size_t size, int nBlockUse, const char *pFileName, int nLine )
+		{
+			Node_t *pNode = (Node_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_NODE_ALIGNMENT, pFileName, nLine );
+			return pNode;
+		}
+
+		static void operator delete( void *p)
+		{
+			MemAlloc_FreeAligned( p );
+		}
+
+		static void operator delete( void *p, int nBlockUse, const char *pFileName, int nLine )
+		{
+			MemAlloc_FreeAligned( p );
+		}
+
 		Node_t() {}
 		Node_t( const T &init ) : elem( init ) {}
 
 		Node_t *pNext;
 		T elem;
-	};
+	} TSLIST_NODE_ALIGN_POST;
 
 	union TSLIST_HEAD_ALIGN NodeLink_t
 	{
+		// override new/delete so we can guarantee 8-byte aligned allocs
+		static void * operator new( size_t size )
+		{
+			NodeLink_t *pNode = (NodeLink_t *)MemAlloc_AllocAlignedFileLine( size, TSLIST_HEAD_ALIGNMENT, __FILE__, __LINE__ );
+			return pNode;
+		}
+
+		static void operator delete( void *p)
+		{
+			MemAlloc_FreeAligned( p );
+		}
+
 		struct Value_t
 		{
 			Node_t *pNode;
-			int32	sequence;
+			intp	sequence;
 		} value;
 
-		int64 value64;
+#ifdef PLATFORM_64BITS
-	};
+		int128 value64x128;
+#else
+		int64 value64x128;
+#endif
+	} TSLIST_HEAD_ALIGN_POST;
 
 	CTSQueue()
 	{
 		COMPILE_TIME_ASSERT( sizeof(Node_t) >= sizeof(TSLNodeBase_t) );
 		if ( ((size_t)&m_Head) % TSLIST_HEAD_ALIGNMENT != 0 )
 		{
-			Error( "CTSQueue: Misaligned queue\n" );
+			Plat_FatalErrorFunc( "CTSQueue: Misaligned queue\n" );
 			DebuggerBreak();
 		}
 		if ( ((size_t)&m_Tail) % TSLIST_HEAD_ALIGNMENT != 0 )
 		{
-			Error( "CTSQueue: Misaligned queue\n" );
+			Plat_FatalErrorFunc( "CTSQueue: Misaligned queue\n" );
 			DebuggerBreak();
 		}
 		m_Count = 0;
@@ -526,7 +736,7 @@ public:
 	{
 		if ( IsDebug() )
 		{
-			Validate();
+			ValidateQueue();
 		}
 
 		Node_t *pNode;
@@ -551,7 +761,7 @@ public:
 	{
 		if ( IsDebug() )
 		{
-			Validate();
+			ValidateQueue();
 		}
 
 		Node_t *pNode;
@@ -561,7 +771,9 @@ public:
 		}
 	}
 
-	bool Validate()
+	bool ValidateQueue()
+	{
+		if ( IsDebug() )
 		{
 			bool bResult = true;
 			int nNodes = 0;
@@ -602,6 +814,11 @@ public:
 
 			return bResult;
 		}
+		else
+		{
+			return true;
+		}
+	}
 
 	void FinishPush( Node_t *pNode, const NodeLink_t &oldTail )
 	{
@@ -610,9 +827,8 @@ public:
 		newTail.value.pNode = pNode;
 		newTail.value.sequence = oldTail.value.sequence + 1;
 
-#ifdef _X360
+		ThreadMemoryBarrier();
-		__lwsync(); // write-release barrier
+
-#endif
 		InterlockedCompareExchangeNodeLink( &m_Tail, newTail, oldTail );
 	}
 
@@ -621,7 +837,7 @@ public:
 #ifdef _DEBUG
 		if ( (size_t)pNode % TSLIST_NODE_ALIGNMENT != 0 )
 		{
-			Error( "CTSListBase: Misaligned node\n" );
+			Plat_FatalErrorFunc( "CTSQueue: Misaligned node\n" );
 			DebuggerBreak();
 		}
 #endif
@@ -632,7 +848,8 @@ public:
 
 		for (;;)
 		{
-			oldTail = m_Tail;
+			oldTail.value.sequence = m_Tail.value.sequence;
+			oldTail.value.pNode = m_Tail.value.pNode;
 			if ( InterlockedCompareExchangeNode( &(oldTail.value.pNode->pNext), pNode, End() ) == End() )
 			{
 				break;
@@ -644,7 +861,7 @@ public:
 			}
 		}
 
-		FinishPush( pNode, oldTail );
+		FinishPush( pNode, oldTail ); // This can fail if another thread pushed between the sequence and node grabs above. Later pushes or pops corrects
 
 		m_Count++;
 
@@ -653,31 +870,32 @@ public:
 
 	Node_t *Pop()
 	{
-		#define TSQUEUE_BAD_NODE_LINK ((Node_t *)0xdeadbeefULL)
+		#define TSQUEUE_BAD_NODE_LINK ( (Node_t *)INT_TO_POINTER( 0xdeadbeef ) )
 		NodeLink_t * volatile		pHead = &m_Head;
 		NodeLink_t * volatile		pTail = &m_Tail;
 		Node_t * volatile *			pHeadNode = &m_Head.value.pNode;
-		volatile int * volatile		pHeadSequence = &m_Head.value.sequence;
+		volatile intp * volatile	pHeadSequence = &m_Head.value.sequence;
 		Node_t * volatile * 		pTailNode = &pTail->value.pNode;
 
 		NodeLink_t head;
 		NodeLink_t newHead;
 		Node_t *pNext;
-		int tailSequence;
+		intp tailSequence;
 		T elem;
 
 		for (;;)
 		{
 			head.value.sequence = *pHeadSequence; // must grab sequence first, which allows condition below to ensure pNext is valid
-#ifdef _X360
+			ThreadMemoryBarrier(); // need a barrier to prevent reordering of these assignments
-			__lwsync(); // 360 needs a barrier to prevent reordering of these assignments
-#endif
 			head.value.pNode	= *pHeadNode;
 			tailSequence		= pTail->value.sequence;
          	pNext				= head.value.pNode->pNext;
 
-			if ( pNext && head.value.sequence == *pHeadSequence ) // Checking pNext only to force optimizer to not reorder the assignment to pNext and the compare of the sequence
+			// Checking pNext only to force optimizer to not reorder the assignment
-			{
+			// to pNext and the compare of the sequence
+			if ( !pNext || head.value.sequence != *pHeadSequence ) 
+				continue;
+
 			if ( bTestOptimizer )
 			{
 				if ( pNext == TSQUEUE_BAD_NODE_LINK )
@@ -686,28 +904,27 @@ public:
 					continue;
 				}
 			}
+
 			if ( head.value.pNode == *pTailNode )
 			{
 				if ( pNext == End() )
-					{
 					return NULL;
-					}
 
 				// Another thread is trying to push, help it along
 				NodeLink_t &oldTail = head; // just reuse local memory for head to build old tail
 				oldTail.value.sequence = tailSequence; // reuse head pNode
 				FinishPush( pNext, oldTail );
+				continue;
 			}
-				else if ( pNext != End() )
+			
+			if ( pNext != End() )
 			{
 				elem = pNext->elem; // NOTE: next could be a freed node here, by design
 				newHead.value.pNode = pNext;
 				newHead.value.sequence = head.value.sequence + 1;
 				if ( InterlockedCompareExchangeNodeLink( pHead, newHead, head ) )
 				{
-#ifdef _X360
+					ThreadMemoryBarrier();
-						__lwsync(); // read-acquire barrier 
-#endif
 					if ( bTestOptimizer )
 					{
 						head.value.pNode->pNext = TSQUEUE_BAD_NODE_LINK;
@@ -716,7 +933,6 @@ public:
 				}
 			}
 		}
-		}
 
 		m_Count--;
 		head.value.pNode->elem = elem;
@@ -747,12 +963,13 @@ public:
 		Node_t *pNode = Pop();
 		if ( !pNode )
 			return false;
+
 		*pResult = pNode->elem;
 		m_FreeNodes.Push( (TSLNodeBase_t *)pNode );
 		return true;
 	}
 
-	int Count()
+	int Count() const
 	{
 		return m_Count;
 	}
@@ -760,7 +977,6 @@ public:
 private:
 	Node_t *End() { return (Node_t *)this; } // just need a unique signifier
 
-#ifndef _WIN64
 	Node_t *InterlockedCompareExchangeNode( Node_t * volatile *ppNode, Node_t *value, Node_t *comperand )
 	{
 		return (Node_t *)::ThreadInterlockedCompareExchangePointer( (void **)ppNode, value, comperand );
@@ -768,41 +984,21 @@ private:
 
 	bool InterlockedCompareExchangeNodeLink( NodeLink_t volatile *pLink, const NodeLink_t &value, const NodeLink_t &comperand )
 	{
-		return ThreadInterlockedAssignIf64( (int64 *)pLink, value.value64, comperand.value64 );
+		return ThreadInterlockedAssignIf64x128( &pLink->value64x128, value.value64x128, comperand.value64x128 );
 	}
 
-#else
-	Node_t *InterlockedCompareExchangeNode( Node_t * volatile *ppNode, Node_t *value, Node_t *comperand )
-	{
-		AUTO_LOCK( m_ExchangeMutex );
-		Node_t *retVal = *ppNode;
-		if ( *ppNode == comperand )
-			*ppNode = value;
-		return retVal;
-	}
-
-	bool InterlockedCompareExchangeNodeLink( NodeLink_t volatile *pLink, const NodeLink_t &value, const NodeLink_t &comperand )
-	{
-		AUTO_LOCK( m_ExchangeMutex );
-		if ( pLink->value64 == comperand.value64 )
-		{
-			pLink->value64  = value.value64;
-			return true;
-		}
-		return false;
-	}
-
-	CThreadFastMutex m_ExchangeMutex;
-#endif
-
 	NodeLink_t m_Head;
 	NodeLink_t m_Tail;
 
 	CInterlockedInt m_Count;
 	
 	CTSListBase m_FreeNodes;
-};
+} TSLIST_HEAD_ALIGN_POST;
 
-#include "tier0/memdbgoff.h"
+#if defined( _WIN32 )
+// Suppress this spurious warning:
+// warning C4700: uninitialized local variable 'oldHead' used
+#pragma warning( pop )
+#endif
 
 #endif // TSLIST_H

public/tier1/bufferstring.h

@@ -153,6 +153,7 @@ public:
 	// Returns NULL if they can't be made relative (on separate drives, for example)
 	DLL_CLASS_IMPORT const char *MakeRelativePath(const char *pFullPath, const char *pDirectory);
 
+	// Copies data from pOther and then purges it
 	DLL_CLASS_IMPORT void MoveFrom(CBufferString &pOther);
 
 	DLL_CLASS_IMPORT void Purge(int nLength);
@@ -242,7 +243,7 @@ public:
 		{
 			m_nAllocated |= ALLOW_HEAP_ALLOCATION;
 		}
-		MoveFrom(const_cast<CBufferStringGrowable&>(other));
+		Insert(0, other.Get());
 	}
 
 	~CBufferStringGrowable()
@@ -259,7 +260,8 @@ public:
 
 	inline CBufferStringGrowable& operator=(const CBufferStringGrowable& src)
 	{
-		MoveFrom(const_cast<CBufferStringGrowable&>(src));
+		Clear();
+		Insert(0, src.Get());
 		return *this;
 	}
 

public/tier1/mempool.h

@@ -34,7 +34,6 @@ enum MemoryPoolGrowType_t
 	UTLMEMORYPOOL_GROW_NONE=0,	// Don't allow new blobs.
 	UTLMEMORYPOOL_GROW_FAST=1,	// New blob size is numElements * (i+1)  (ie: the blocks it allocates get larger and larger each time it allocates one).
 	UTLMEMORYPOOL_GROW_SLOW=2,	// New blob size is numElements.
-	UTLMEMORYPOOL_GROW_RBTREE=3	// No blobs. All blocks are stored in RBTree.
 };
 
 class CUtlMemoryPoolBase
@@ -51,22 +50,20 @@ public:
 	DLL_CLASS_IMPORT void		Free( void *pMem );
 	
 	// Frees everything
-	DLL_CLASS_IMPORT void		Clear();
+	void Clear() { ClearDestruct( NULL ); }
 	
 	// returns number of allocated blocks
 	int Count() const		{ return m_BlocksAllocated; }
 	int PeakCount() const	{ return m_PeakAlloc; }
 	int BlockSize() const	{ return m_BlockSize; }
-	int Size() const		{ return m_NumBlobs * m_BlocksPerBlob * m_BlockSize; }
+	int Size() const		{ return m_TotalSize; }
 
 	DLL_CLASS_IMPORT bool		IsAllocationWithinPool( void *pMem ) const;
 
 protected:
-	struct FreeList_t
+	DLL_CLASS_IMPORT void		ClearDestruct( void (*)( void* ) );
-	{
-		FreeList_t *m_pNext;
-	};
 
+private:
 	class CBlob
 	{
 	public:
@@ -76,10 +73,8 @@ protected:
 		char	m_Padding[3]; // to int align the struct
 	};
 
-	DLL_CLASS_IMPORT FreeList_t*	AddNewBlob();
+	DLL_CLASS_IMPORT bool AddNewBlob();
 	DLL_CLASS_IMPORT void ResetAllocationCounts();
-	DLL_CLASS_IMPORT void			InternalClear( CBlob *blob, FreeList_t *free_list );
-	DLL_CLASS_IMPORT void			ClearDestruct( void (*)( void* ) );
 
 	int			m_BlockSize;
 	int			m_BlocksPerBlob;
@@ -91,15 +86,14 @@ protected:
 	unsigned short	m_nAlignment;
 	unsigned short	m_NumBlobs;
 	
-	FreeList_t**	m_ppTailOfFreeList;
+	CTSListBase		m_FreeBlocks;
-	FreeList_t*		m_pHeadOfFreeList;
-
-	CBlob**			m_ppBlobTail;
-	CBlob*			m_pBlobHead;
 	
 	MemAllocAttribute_t m_AllocAttribute;
 	
 	bool 			m_Unk1;
+	CThreadMutex	m_Mutex;
+	CBlob*			m_pBlobHead;
+	int				m_TotalSize;
 };
 
 

public/tier1/utldict.h

@@ -72,6 +72,9 @@ public:
 	// Number of elements
 	unsigned int Count() const;
 	
+	// Number of allocated slots
+	I MaxElement() const;
+	
 	// Checks if a node is valid and in the tree
 	bool  IsValidIndex( I i ) const;
 	
@@ -190,6 +193,14 @@ inline unsigned int CUtlDict<T, I, COMPARE_TYPE>::Count() const
 	return m_Elements.Count(); 
 }
 
+//-----------------------------------------------------------------------------
+// Number of allocated slots
+//-----------------------------------------------------------------------------
+template <class T, class I, int COMPARE_TYPE>
+inline I CUtlDict<T, I, COMPARE_TYPE>::MaxElement() const
+{
+	return m_Elements.MaxElement();
+}
 
 //-----------------------------------------------------------------------------
 // Checks if a node is valid and in the tree

tier1/keyvalues3.cpp

@@ -1796,10 +1796,7 @@ void CKeyValues3Context::CopyMetaData( KV3MetaData_t* pDest, const KV3MetaData_t
 
 	FOR_EACH_MAP_FAST( pSrc->m_Comments, iter )
 	{
-		CBufferStringGrowable<8> buff;
+		pDest->m_Comments.Insert( pSrc->m_Comments.Key( iter ), pSrc->m_Comments.Element( iter ) );
-		buff.Insert( 0, pSrc->m_Comments[ iter ].Get() );
-
-		pDest->m_Comments.Insert( pSrc->m_Comments.Key( iter ), buff );
 	}
 }