1

filesystem/filesystem_stdio.vpc

@@ -0,0 +1,107 @@
+//-----------------------------------------------------------------------------
+//	FILESYSTEM_STDIO.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$macro SRCDIR		".."
+$Macro OUTBINDIR	"$SRCDIR\..\game\bin"
+
+$include "$SRCDIR\vpc_scripts\source_dll_base.vpc"
+$include "$SRCDIR\vpc_scripts\source_cryptlib_include.vpc"
+
+$Configuration	"Debug"
+{
+	$General
+	{
+		$OutputDirectory		"Debug_Stdio"     [$WINDOWS]
+		$IntermediateDirectory	"Debug_Stdio"     [$WINDOWS]
+		$OutputDirectory		"Debug_Stdio_360" [$X360]
+		$IntermediateDirectory	"Debug_Stdio_360" [$X360]
+	}
+}
+
+$Configuration	"Release"
+{
+	$General
+	{
+		$OutputDirectory		"Release_Stdio"     [$WINDOWS]
+		$IntermediateDirectory	"Release_Stdio"     [$WINDOWS]
+		$OutputDirectory		"Release_Stdio_360" [$X360]
+		$IntermediateDirectory	"Release_Stdio_360" [$X360]
+	}
+}
+
+$Configuration
+{
+	$Compiler
+	{
+		$PreprocessorDefinitions	"$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE" [$WIN64]
+		$PreprocessorDefinitions	"$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE;_USE_32BIT_TIME_T" [!$WIN64]
+
+		// Enable super-fun workaround for using MSVC2015 to target Windows XP
+		// https://connect.microsoft.com/VisualStudio/feedback/details/1600505/stat-not-working-on-windows-xp-using-v14-xp-platform-toolset-vs2015
+		$PreprocessorDefinitions	"$BASE;FILESYSTEM_MSVC2015_STAT_BUG_WORKAROUND" [$VS2015]
+
+		// Add VPK support
+		$PreprocessorDefinitions	"$BASE;SUPPORT_PACKED_STORE" [!$WIN64]
+	}
+	$Linker
+	{
+			$SystemLibraries                                                        "iconv" [$OSXALL]
+	}
+}
+
+$Project "FileSystem_Stdio"
+{
+	$Folder	"Source Files"
+	{
+		$File	"basefilesystem.cpp"
+		$File	"packfile.cpp"
+		$File	"filetracker.cpp"
+		$File	"filesystem_async.cpp"
+		$File	"filesystem_stdio.cpp"
+		$File	"$SRCDIR\public\kevvaluescompiler.cpp"
+		$File	"$SRCDIR\public\zip_utils.cpp"
+		$File	"QueuedLoader.cpp"
+		$File	"linux_support.cpp"			[$POSIX]
+	}
+
+
+	$Folder	"Header Files"
+	{
+		$File	"basefilesystem.h"
+		$File	"packfile.h"
+		$File	"filetracker.h"
+		$File	"threadsaferefcountedobject.h"
+		$File	"$SRCDIR\public\tier0\basetypes.h"
+		$File	"$SRCDIR\public\bspfile.h"
+		$File	"$SRCDIR\public\bspflags.h"
+		$File	"$SRCDIR\public\mathlib\bumpvects.h"
+		$File	"$SRCDIR\public\tier1\characterset.h"
+		$File	"$SRCDIR\public\tier0\dbg.h"
+		$File	"$SRCDIR\public\tier0\fasttimer.h"
+		$File	"$SRCDIR\public\filesystem.h"
+		$File	"$SRCDIR\public\ifilelist.h"
+		$File	"$SRCDIR\public\appframework\IAppSystem.h"
+		$File	"$SRCDIR\public\tier1\interface.h"
+		$File	"$SRCDIR\public\mathlib\mathlib.h"
+		$File	"$SRCDIR\public\tier0\platform.h"
+		$File	"$SRCDIR\public\tier1\strtools.h"
+		$File	"$SRCDIR\public\tier1\utlmemory.h"
+		$File	"$SRCDIR\public\tier1\utlrbtree.h"
+		$File	"$SRCDIR\public\tier1\utlsymbol.h"
+		$File	"$SRCDIR\public\tier1\utlvector.h"
+		$File	"$SRCDIR\public\mathlib\vector.h"
+		$File	"$SRCDIR\public\mathlib\vector2d.h"
+		$File	"$SRCDIR\public\mathlib\vector4d.h"
+		$File	"$SRCDIR\public\vstdlib\vstdlib.h"
+		$File	"$SRCDIR\public\keyvaluescompiler.h"
+		$File	"$SRCDIR\public\filesystem\IQueuedLoader.h"
+	}
+
+	$Folder "Link Libraries"
+	{
+		$Lib	tier2
+	}
+}

filesystem/filesystem_stdio/ThreadSafeRefCountedObject.h

@@ -0,0 +1,85 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef THREADSAFEREFCOUNTEDOBJECT_H
+#define THREADSAFEREFCOUNTEDOBJECT_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+// This class can be used for fast access to an object from multiple threads,
+// and the main thread can wait until the threads are done using the object before it frees the object.
+template< class T >
+class CThreadSafeRefCountedObject
+{
+public:
+	CThreadSafeRefCountedObject( T initVal )
+	{
+		m_RefCount = 0;
+		m_pObject = initVal;
+		m_RefCount = 0;
+	}
+	
+	void Init( T pObj )
+	{
+		Assert( ThreadInMainThread() );
+		Assert( !m_pObject );
+		m_RefCount = 0;
+		m_pObject = pObj;
+		m_RefCount = 1;
+	}
+	
+	// Threads that access the object need to use AddRef/Release to access it.
+	T AddRef()
+	{
+		if ( ++m_RefCount > 1 )
+		{
+			return m_pObject;
+		}
+		else
+		{
+			// If the refcount was 0 when we called this, then the whitelist is about to be freed.
+			--m_RefCount;
+			return NULL;
+		}
+	}
+	void ReleaseRef( T pObj )
+	{
+		if ( --m_RefCount >= 1 )
+		{
+			Assert( m_pObject == pObj );
+		}
+	}
+
+	// The main thread can use this to access the object, since only it can Init() and Free() the object.
+	T GetInMainThread()
+	{
+		Assert( ThreadInMainThread() );
+		return m_pObject;
+	}
+
+	// The main thread calls this after it has released its last reference to the object.
+	void ResetWhenNoRemainingReferences( T newValue )
+	{
+		Assert( ThreadInMainThread() );
+		
+		// Wait until we can free it.
+		while ( m_RefCount > 0 )
+		{
+			CThread::Sleep( 20 );
+		}
+		
+		m_pObject = newValue;
+	}
+
+private:	
+	CInterlockedIntT<long>	m_RefCount;
+	T						m_pObject;
+};
+
+
+#endif // THREADSAFEREFCOUNTEDOBJECT_H

filesystem/filesystem_steam.vpc

@@ -0,0 +1,80 @@
+//-----------------------------------------------------------------------------
+//	FILESYSTEM_STEAM.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR           ".."
+$Macro OUTBINDIR	"$SRCDIR\..\game\bin"
+
+$Include "$SRCDIR\vpc_scripts\source_dll_base.vpc"
+
+$Configuration
+{
+	$Compiler
+	{
+		$AdditionalIncludeDirectories		"$SRCDIR\public,$SRCDIR\public\tier1,$SRCDIR\common"
+		$PreprocessorDefinitions			"$BASE;FILESYSTEM_STDIO_EXPORTS;DONT_PROTECT_FILEIO_FUNCTIONS;PROTECTED_THINGS_ENABLE;FILESYSTEM_STEAM"
+	}
+	$Linker
+	{
+			$SystemLibraries                                                        "iconv" [$OSXALL]
+	}
+}
+
+$Project "FileSystem_Steam"
+{
+	$Folder	"Source Files"
+	{
+		$File	"basefilesystem.cpp"
+		$File	"packfile.cpp"
+		$File	"filetracker.cpp"
+		$File	"filesystem_async.cpp"
+		$File	"filesystem_steam.cpp"
+		$File	"linux_support.cpp" [$POSIX]
+		$File	"QueuedLoader.cpp"
+		$File	"$SRCDIR\public\kevvaluescompiler.cpp"
+		$File	"$SRCDIR\public\keyvaluescompiler.h"
+		$File	"$SRCDIR\public\zip_utils.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"basefilesystem.h"
+		$File	"packfile.h"
+		$File	"filetracker.h"
+		$File	"threadsaferefcountedobject.h"
+		$File	"$SRCDIR\public\tier0\basetypes.h"
+		$File	"$SRCDIR\public\bspfile.h"
+		$File	"$SRCDIR\public\bspflags.h"
+		$File	"$SRCDIR\public\mathlib\bumpvects.h"
+		$File	"$SRCDIR\public\tier1\characterset.h"
+		$File	"$SRCDIR\public\tier0\dbg.h"
+		$File	"$SRCDIR\public\tier0\fasttimer.h"
+		$File	"$SRCDIR\public\filesystem.h"
+		$File	"$SRCDIR\public\ifilelist.h"
+		$File	"$SRCDIR\public\appframework\IAppSystem.h"
+		$File	"$SRCDIR\public\tier1\interface.h"
+		$File	"$SRCDIR\public\mathlib\mathlib.h"
+		$File	"$SRCDIR\public\tier0\platform.h"
+		$File	"$SRCDIR\public\tier1\strtools.h"
+		$File	"$SRCDIR\public\tier1\utlmemory.h"
+		$File	"$SRCDIR\public\tier1\utlrbtree.h"
+		$File	"$SRCDIR\public\tier1\utlsymbol.h"
+		$File	"$SRCDIR\public\tier1\utlvector.h"
+		$File	"$SRCDIR\public\mathlib\vector.h"
+		$File	"$SRCDIR\public\mathlib\vector2d.h"
+		$File	"$SRCDIR\public\mathlib\vector4d.h"
+		$File	"$SRCDIR\public\vstdlib\vstdlib.h"
+		$File	"$SRCDIR\public\zip_utils.h"
+	}
+
+	$Folder	"Link Libraries" 
+	{
+		$Lib tier2 
+
+		$DynamicFile	"$SRCDIR\lib\public\steam_api.lib" [$WIN32]
+		$DynamicFile	"$SRCDIR\lib\$PLATFORM\$_IMPLIB_PREFIXsteam$_IMPLIB_EXT" [$POSIX || $LINUX]
+		$DynamicFile	"$SRCDIR\lib\$PLATFORM\$_IMPLIB_PREFIXsteam_api$_IMPLIB_EXT" [$POSIX || $LINUX]
+	}
+}

filesystem/filetracker.cpp

@@ -0,0 +1,596 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#include "basefilesystem.h"
+#include "tier0/vprof.h"
+
+// NOTE: This has to be the last file included!
+#include "tier0/memdbgon.h"
+
+#if !defined( DEDICATED )
+
+#ifdef SUPPORT_PACKED_STORE
+
+unsigned ThreadStubProcessMD5Requests( void *pParam )
+{
+	return ((CFileTracker2 *)pParam)->ThreadedProcessMD5Requests();
+}
+
+//-----------------------------------------------------------------------------
+// ThreadedProcessMD5Requests
+// Calculate the MD5s of all the blocks submitted to us
+//-----------------------------------------------------------------------------
+unsigned CFileTracker2::ThreadedProcessMD5Requests()
+{
+	ThreadSetDebugName( "ProcessMD5Requests" );
+
+	while ( m_bThreadShouldRun )
+	{
+		StuffToMD5_t stuff;
+
+		while ( m_PendingJobs.PopItem( &stuff ) )
+		{
+			tmZone( TELEMETRY_LEVEL0, TMZF_NONE, "%s", __FUNCTION__ ); 
+
+			MD5Context_t ctx;
+			memset( &ctx, 0, sizeof(MD5Context_t) );
+			MD5Init( &ctx );
+			MD5Update( &ctx, stuff.m_pubBuffer, stuff.m_cubBuffer );
+			MD5Final( stuff.m_md5Value.bits, &ctx);
+
+			{
+				// update the FileTracker MD5 database
+				AUTO_LOCK( m_Mutex );
+
+				TrackedVPKFile_t &trackedVPKFile = m_treeTrackedVPKFiles[ stuff.m_idxTrackedVPKFile ];
+				TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ trackedVPKFile.m_idxAllOpenedFiles ];
+
+				memcpy( trackedfile.m_filehashFinal.m_md5contents.bits, stuff.m_md5Value.bits, sizeof( trackedfile.m_filehashFinal.m_md5contents.bits ) );
+				trackedfile.m_filehashFinal.m_crcIOSequence = stuff.m_cubBuffer;
+				trackedfile.m_filehashFinal.m_cbFileLen = stuff.m_cubBuffer;
+				trackedfile.m_filehashFinal.m_eFileHashType = FileHash_t::k_EFileHashTypeEntireFile;
+				trackedfile.m_filehashFinal.m_nPackFileNumber = trackedVPKFile.m_nPackFileNumber;
+				trackedfile.m_filehashFinal.m_PackFileID = trackedVPKFile.m_PackFileID;
+			}
+
+			m_CompletedJobs.PushItem( stuff );
+			m_threadEventWorkCompleted.Set();
+		}
+
+		{
+			tmZone( TELEMETRY_LEVEL0, TMZF_IDLE, "m_threadEventWorkToDo" ); 
+
+			m_threadEventWorkToDo.Wait( 1000 );
+		}
+	}
+
+	return 0;
+}
+
+//-----------------------------------------------------------------------------
+// SubmitThreadedMD5Request
+// add pubBuffer,cubBuffer to our queue of stuff to MD5
+// caller promises that the memory will remain valid
+// until BlockUntilMD5RequestComplete() is called
+// returns: request handle
+//-----------------------------------------------------------------------------
+int CFileTracker2::SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction )
+{
+	int idxList;
+	StuffToMD5_t stuff;
+
+	{
+		AUTO_LOCK( m_Mutex );
+
+		TrackedVPKFile_t trackedVPKFileFind;
+		trackedVPKFileFind.m_nPackFileNumber = nPackFileNumber;
+		trackedVPKFileFind.m_PackFileID = PackFileID;
+		trackedVPKFileFind.m_nFileFraction = nPackFileFraction;
+
+		int idxTrackedVPKFile = m_treeTrackedVPKFiles.Find( trackedVPKFileFind );
+		if ( idxTrackedVPKFile != m_treeTrackedVPKFiles.InvalidIndex() )
+		{
+			// dont early out if we have already done the MD5, if the caller wants us
+			// to do it again - then do it again
+			m_cDupMD5s++;
+		}
+		else
+		{
+			// this is an error, we should already know about the file
+			Assert(0);
+			return 0;
+		}
+
+		SubmittedMd5Job_t submittedjob;
+		submittedjob.m_bFinished = false;
+		idxList = m_SubmittedJobs.AddToTail( submittedjob );
+
+		stuff.m_pubBuffer = pubBuffer;
+		stuff.m_cubBuffer = cubBuffer;
+		stuff.m_idxTrackedVPKFile = idxTrackedVPKFile;
+		stuff.m_idxListSubmittedJobs = idxList;
+	}
+
+	// Start thread if it wasn't already active. Do this down here due to the
+	//  return 0 above us. Ie, don't start the thread unless we actually have work
+	//  to do.
+	if ( m_hWorkThread == NULL )
+	{
+		Assert( !m_bThreadShouldRun );
+		m_bThreadShouldRun = true;
+		m_hWorkThread = CreateSimpleThread( ThreadStubProcessMD5Requests, this );
+	}
+
+	// submit the work
+	m_PendingJobs.PushItem( stuff );
+	m_threadEventWorkToDo.Set();
+
+	return idxList + 1;
+}
+
+//-----------------------------------------------------------------------------
+// IsMD5RequestComplete
+// is request identified by iRequest finished?
+// ( the caller wants to free the memory, but now must wait until we finish
+// calculating the MD5 )
+//-----------------------------------------------------------------------------
+bool CFileTracker2::IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut )
+{
+	AUTO_LOCK( m_Mutex );
+	int idxListWaiting = iRequest - 1;
+
+	// deal with all completed jobs
+	StuffToMD5_t stuff;
+	while ( m_CompletedJobs.PopItem( &stuff ) )
+	{
+		int idxList = stuff.m_idxListSubmittedJobs;
+		Q_memcpy( &m_SubmittedJobs[ idxList ].m_md5Value, &stuff.m_md5Value, sizeof( MD5Value_t ) );
+		m_SubmittedJobs[ idxList ].m_bFinished = true;
+	}
+
+	// did the one we wanted finish?
+	if ( m_SubmittedJobs[ idxListWaiting ].m_bFinished )
+	{
+		Q_memcpy( pMd5ValueOut, &m_SubmittedJobs[ idxListWaiting ].m_md5Value, sizeof( MD5Value_t ) );
+		// you can not ask again, we have removed it from the list
+		m_SubmittedJobs.Remove(idxListWaiting);
+		return true;
+	}
+
+	// not done yet
+	return false;
+}
+
+//-----------------------------------------------------------------------------
+// BlockUntilMD5RequestComplete
+// block until request identified by iRequest is finished
+// ( the caller wants to free the memory, but now must wait until we finish
+// calculating the MD5 )
+//-----------------------------------------------------------------------------
+bool CFileTracker2::BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut )
+{
+	while ( 1 )
+	{
+		if ( IsMD5RequestComplete( iRequest, pMd5ValueOut ) )
+			return true;
+		m_cThreadBlocks++;
+		m_threadEventWorkCompleted.Wait( 1 );
+	}
+	return false;
+}
+
+#endif // SUPPORT_PACKED_STORE
+
+CFileTracker2::CFileTracker2( CBaseFileSystem *pFileSystem ):
+	m_treeAllOpenedFiles( TrackedFile_t::Less ),
+	m_treeTrackedVPKFiles( TrackedVPKFile_t::Less )
+{
+#if defined( DEDICATED )
+    Assert( 0 );
+#endif
+
+	m_pFileSystem = pFileSystem;
+
+	m_cThreadBlocks = 0;
+	m_cDupMD5s = 0;
+
+#ifdef SUPPORT_PACKED_STORE
+	m_bThreadShouldRun = false;
+	m_hWorkThread = NULL;
+#endif
+}
+
+CFileTracker2::~CFileTracker2()
+{
+#ifdef SUPPORT_PACKED_STORE
+	Assert( !m_bThreadShouldRun );
+	Assert( m_hWorkThread == NULL );
+#endif
+}
+
+void CFileTracker2::ShutdownAsync()
+{
+#ifdef SUPPORT_PACKED_STORE
+	m_bThreadShouldRun = false;
+	m_threadEventWorkToDo.Set();
+	// wait for it to die
+	if ( m_hWorkThread )
+	{
+		ThreadJoin( m_hWorkThread );
+		ReleaseThreadHandle( m_hWorkThread );
+		m_hWorkThread = NULL;
+	}
+#endif
+}
+
+void CFileTracker2::MarkAllCRCsUnverified()
+{
+	// AUTO_LOCK( m_Mutex );
+}
+
+int CFileTracker2::GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles )
+{
+	return 0;
+}
+
+EFileCRCStatus CFileTracker2::CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash )
+{
+	Assert( ThreadInMainThread() );
+	AUTO_LOCK( m_Mutex );
+
+	TrackedFile_t trackedfileFind;
+	trackedfileFind.RebuildFileName( m_stringPool, pRelativeFilename, pPathID, nFileFraction );
+
+	int idx = m_treeAllOpenedFiles.Find( trackedfileFind );
+	if ( idx != m_treeAllOpenedFiles.InvalidIndex() )
+	{
+		TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idx ];
+
+		if ( trackedfile.m_bFileInVPK )
+		{
+			// the FileHash is not meaningful, because the file is in a VPK, we have hashed the entire VPK
+			// if the user is sending us a hash for this file, it means he has extracted it from the VPK and tricked the client into loading it
+			// instead of the version in the VPK.
+			return k_eFileCRCStatus_FileInVPK;
+		}
+
+		return k_eFileCRCStatus_CantOpenFile;
+	}
+	else
+	{
+		return k_eFileCRCStatus_CantOpenFile;
+	}
+}
+
+void TrackedFile_t::RebuildFileName( CStringPool &stringPool, const char *pFilename, const char *pPathID, int nFileFraction )
+{
+	char szFixedName[ MAX_PATH ];
+	char szPathName[ MAX_PATH ];
+
+	V_strcpy_safe( szFixedName, pFilename );
+	V_RemoveDotSlashes( szFixedName );
+	V_FixSlashes( szFixedName );
+	V_strlower( szFixedName ); // !KLUDGE!
+	m_filename = stringPool.Allocate( szFixedName );
+
+	V_strcpy_safe( szPathName, pPathID ? pPathID : "" );
+	V_strupr( szPathName ); // !KLUDGE!
+	m_path = stringPool.Allocate( szPathName );
+
+	// CRC32_t crcFilename;
+	// CRC32_Init( &crcFilename );
+	// CRC32_ProcessBuffer( &crcFilename, m_filename, Q_strlen( m_filename ) );
+	// CRC32_ProcessBuffer( &crcFilename, m_path, Q_strlen( m_path ) );
+	// CRC32_Final( &crcFilename );
+
+	// m_crcIdentifier = crcFilename;
+
+	m_nFileFraction = nFileFraction;
+}
+
+#ifdef SUPPORT_PACKED_STORE
+
+void CFileTracker2::NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle )
+{
+#if !defined( _GAMECONSOLE ) && !defined( DEDICATED )
+	AUTO_LOCK( m_Mutex );
+	Assert( iSearchPathStoreId > 0 );
+
+	int idxFile = IdxFileFromName( pFilename, pPathID, 0, false );
+	TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ];
+
+	// we could use the CRC data from the VPK header - and verify it
+	// VPKHandle.GetFileCRCFromHeaderData();
+	// for now all we are going to do is track that this file came from a VPK
+	trackedfile.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID;
+	trackedfile.m_nPackFileNumber = VPKHandle.m_nFileNumber; // this might be useful to send up
+	trackedfile.m_iLoadedSearchPathStoreId = iSearchPathStoreId;
+	trackedfile.m_bFileInVPK = true;
+#endif // !defined( _GAMECONSOLE ) && !defined( DEDICATED )
+}
+
+#endif // SUPPORT_PACKED_STORE
+
+struct FileListToUnloadForWhitelistChange : public IFileList
+{
+	virtual bool IsFileInList( const char *pFilename )
+	{
+		char szFixedName[ MAX_PATH ];
+		GetFixedName( pFilename, szFixedName );
+		return m_dictFiles.Find( szFixedName ) >= 0;
+	}
+
+	virtual void Release()
+	{
+		delete this;
+	}
+
+	void AddFile( const char *pszFilename )
+	{
+		char szFixedName[ MAX_PATH ];
+		GetFixedName( pszFilename, szFixedName );
+		if ( m_dictFiles.Find( szFixedName ) < 0 )
+			m_dictFiles.Insert( szFixedName );
+	}
+
+	void GetFixedName( const char *pszFilename, char *pszFixedName )
+	{
+		V_strncpy( pszFixedName, pszFilename, MAX_PATH );
+		V_strlower( pszFixedName );
+		V_FixSlashes( pszFixedName );
+	}
+
+	CUtlDict<int> m_dictFiles;
+};
+
+IFileList *CFileTracker2::GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList )
+{
+	FileListToUnloadForWhitelistChange *pResult = new FileListToUnloadForWhitelistChange;
+
+	for ( int i = m_treeAllOpenedFiles.FirstInorder() ; i >= 0 ; i = m_treeAllOpenedFiles.NextInorder( i ) )
+	{
+		TrackedFile_t &f = m_treeAllOpenedFiles[i];
+
+		// !KLUDGE! If we ignored it at all, just reload it.
+		// This is more conservative than we need to be, but the set of files we are ignoring is probably
+		// pretty small so it should be fine.
+		if ( f.m_bIgnoredForPureServer )
+		{
+			f.m_bIgnoredForPureServer = false;
+#ifdef PURE_SERVER_DEBUG_SPEW
+			Msg( "%s was ignored for pure server purposes.  Queuing for reload\n", f.m_filename );
+#endif
+			pResult->AddFile( f.m_filename );
+			continue;
+		}
+
+		if ( f.m_iLoadedSearchPathStoreId != 0 && pNewWhiteList && pNewWhiteList->GetFileClass( f.m_filename ) == ePureServerFileClass_AnyTrusted )
+		{
+			// Check if we loaded it from a path that no longer exists or is no longer trusted
+			const CBaseFileSystem::CSearchPath *pSearchPath = m_pFileSystem->FindSearchPathByStoreId( f.m_iLoadedSearchPathStoreId );
+			if ( pSearchPath == NULL )
+			{
+#ifdef PURE_SERVER_DEBUG_SPEW
+				Msg( "%s was loaded from search path that's no longer mounted.  Queuing for reload\n", f.m_filename );
+#endif
+				pResult->AddFile( f.m_filename );
+			}
+			else if ( !pSearchPath->m_bIsTrustedForPureServer )
+			{
+#ifdef PURE_SERVER_DEBUG_SPEW
+				Msg( "%s was loaded from search path that's not currently trusted.  Queuing for reload\n", f.m_filename );
+#endif
+				pResult->AddFile( f.m_filename );
+			}
+			else
+			{
+#if defined( _DEBUG ) && defined( PURE_SERVER_DEBUG_SPEW )
+				Msg( "%s is OK.  Keeping\n", f.m_filename );
+#endif
+			}
+		}
+	}
+
+	// Do we need to reload anything?
+	if ( pResult->m_dictFiles.Count() > 0 )
+		return pResult;
+
+	// Nothing to reload, return an empty list as an optimization
+	pResult->Release();
+	return NULL;
+}
+
+#ifdef SUPPORT_PACKED_STORE
+
+void CFileTracker2::AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &VPKHandle )
+{
+#if !defined( DEDICATED )
+	AUTO_LOCK( m_Mutex );
+
+	char szDataFileName[MAX_PATH];
+	VPKHandle.m_nFileNumber = nPackFileNumber;
+	VPKHandle.GetPackFileName( szDataFileName, sizeof(szDataFileName) );
+	const char *pszFileName = V_GetFileName( szDataFileName );
+
+	TrackedVPKFile_t trackedVPKFile;
+	trackedVPKFile.m_nPackFileNumber = VPKHandle.m_nFileNumber;
+	trackedVPKFile.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID;
+	trackedVPKFile.m_nFileFraction = nFileFraction;
+	trackedVPKFile.m_idxAllOpenedFiles = IdxFileFromName( pszFileName, "GAME", nFileFraction, true );
+
+	m_treeTrackedVPKFiles.Insert( trackedVPKFile );
+
+	TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ trackedVPKFile.m_idxAllOpenedFiles ];
+    // These set in IdxFileFromName:
+    //   trackedfile.m_crcIdentifier
+    //   trackedfile.m_filename
+    //   trackedfile.m_path
+    //   trackedfile.m_bPackOrVPKFile
+    //   trackedfile.m_nFileFraction
+    // Not set:
+    //   trackedfile.m_iLoadedSearchPathStoreId
+    //   trackedfile.m_bIgnoredForPureServer
+	trackedfile.m_bFileInVPK = false;
+	trackedfile.m_bPackOrVPKFile = true;
+	trackedfile.m_filehashFinal.m_cbFileLen = cbFileLen;
+	trackedfile.m_filehashFinal.m_eFileHashType = FileHash_t::k_EFileHashTypeEntireFile;
+	trackedfile.m_filehashFinal.m_nPackFileNumber = nPackFileNumber;
+	trackedfile.m_filehashFinal.m_PackFileID = VPKHandle.m_pOwner->m_PackFileID;
+	trackedfile.m_filehashFinal.m_crcIOSequence = cbFileLen;
+	Q_memcpy( trackedfile.m_filehashFinal.m_md5contents.bits, md5.bits, sizeof( md5.bits) );
+#endif // !DEDICATED
+}
+
+#endif // SUPPORT_PACKED_STORE
+
+int CFileTracker2::IdxFileFromName( const char *pFilename, const char *pPathID, int nFileFraction, bool bPackOrVPKFile )
+{
+	TrackedFile_t trackedfile;
+
+	trackedfile.RebuildFileName( m_stringPool, pFilename, pPathID, nFileFraction );
+	trackedfile.m_bPackOrVPKFile = bPackOrVPKFile;
+
+	int idxFile = m_treeAllOpenedFiles.Find( trackedfile );
+	if ( idxFile == m_treeAllOpenedFiles.InvalidIndex() )
+	{
+		idxFile = m_treeAllOpenedFiles.Insert( trackedfile );
+	}
+
+	return idxFile;
+}
+
+#ifdef SUPPORT_PACKED_STORE
+
+int CFileTracker2::NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength )
+{
+#if !defined( _GAMECONSOLE )
+	AUTO_LOCK( m_Mutex );
+
+	int idxFile = IdxFileFromName( pVPKAbsPath, pPathID, 0, true );
+
+	TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ];
+	// we have the real name we want to use. correct the name
+	trackedfile.m_bPackOrVPKFile = true;
+	trackedfile.m_PackFileID = idxFile + 1;
+	trackedfile.m_filehashFinal.m_PackFileID = trackedfile.m_PackFileID;
+	trackedfile.m_filehashFinal.m_nPackFileNumber = -1;
+	m_treeAllOpenedFiles.Reinsert( idxFile );
+	return idxFile + 1;
+#else
+	return 0;
+#endif
+}
+
+#endif // SUPPORT_PACKED_STORE
+
+void CFileTracker2::NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId )
+{
+#if !defined( _GAMECONSOLE )
+	AUTO_LOCK( m_Mutex );
+
+	int idxFile = IdxFileFromName( pFilename, pPathID, 0, false );
+	m_treeAllOpenedFiles[ idxFile ].m_bIgnoredForPureServer = true;
+#endif
+}
+
+void CFileTracker2::NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength )
+{
+#if !defined( _GAMECONSOLE ) && !defined( DEDICATED )
+	AUTO_LOCK( m_Mutex );
+
+	Assert( iSearchPathStoreId != 0 );
+
+	int idxFile = IdxFileFromName( pFilename, pPathID, 0, false );
+	TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ];
+	trackedfile.m_iLoadedSearchPathStoreId = iSearchPathStoreId;
+#endif
+}
+
+void CFileTracker2::NoteFileUnloaded( const char *pFilename, const char *pPathID )
+{
+#if !defined( _GAMECONSOLE )
+	AUTO_LOCK( m_Mutex );
+
+	// Locate bookeeping entry, if any
+	TrackedFile_t trackedfile;
+	trackedfile.RebuildFileName( m_stringPool, pFilename, pPathID, 0 );
+
+	int idxFile = m_treeAllOpenedFiles.Find( trackedfile );
+	if ( idxFile >= 0 )
+	{
+		// Clear state
+		TrackedFile_t &trackedfile = m_treeAllOpenedFiles[ idxFile ];
+		trackedfile.m_iLoadedSearchPathStoreId = 0;
+		trackedfile.m_bIgnoredForPureServer = false;
+	}
+#endif
+}
+
+int CFileTracker2::ListOpenedFiles( bool bAllOpened, const char *pchFilenameFind )
+{
+	AUTO_LOCK( m_Mutex );
+
+	int i;
+	int InvalidIndex;
+
+	if ( bAllOpened )
+	{
+		i = m_treeAllOpenedFiles.FirstInorder();
+		InvalidIndex = m_treeAllOpenedFiles.InvalidIndex();
+	}
+	else
+	{
+		i = m_treeTrackedVPKFiles.FirstInorder();
+		InvalidIndex = m_treeTrackedVPKFiles.InvalidIndex();
+	}
+
+	Msg( "#, Path, FileName, (PackFileID, PackFileNumber), FileLen, FileFraction\n" );
+
+	int count = 0;
+	int cPackFiles = 0;
+	while ( i != InvalidIndex )
+	{
+		int index = bAllOpened ? i : m_treeTrackedVPKFiles[ i ].m_idxAllOpenedFiles;
+		TrackedFile_t &file = m_treeAllOpenedFiles[ index ];
+
+		if ( file.m_PackFileID )
+			cPackFiles++;
+		if ( !pchFilenameFind ||
+			Q_strstr( file.m_filename, pchFilenameFind ) ||
+			Q_strstr( file.m_path, pchFilenameFind ) )
+		{
+			Msg( "%d %s %s ( %d, %d ) %d %d%s%s\n", 
+				count, file.m_path, file.m_filename, file.m_PackFileID, file.m_nPackFileNumber,
+				file.m_filehashFinal.m_cbFileLen, file.m_nFileFraction /*, file.m_crcIdentifier*/,
+				file.m_bFileInVPK ? " (invpk)" : "",
+				file.m_bPackOrVPKFile ? " (vpk)" : "");
+		}
+
+		i = bAllOpened ? m_treeAllOpenedFiles.NextInorder( i ) : m_treeTrackedVPKFiles.NextInorder( i );
+		count++;
+	}
+
+	Msg( "cThreadedBlocks:%d cDupMD5s:%d\n", m_cThreadBlocks, m_cDupMD5s );
+	Msg( "TrackedVPKFiles:%d AllOpenedFiles:%d files VPKfiles:%d StringPoolCount:%d\n",
+		m_treeTrackedVPKFiles.Count(), m_treeAllOpenedFiles.Count(), cPackFiles, m_stringPool.Count() );
+	return m_treeAllOpenedFiles.Count();
+}
+
+static void CC_TrackerListAllFiles( const CCommand &args )
+{
+	const char *pchFilenameFind = ( args.ArgC() >= 2 ) ? args[1] : NULL;
+	BaseFileSystem()->m_FileTracker2.ListOpenedFiles( true, pchFilenameFind );
+}
+static ConCommand trackerlistallfiles( "trackerlistallfiles", CC_TrackerListAllFiles, "TrackerListAllFiles" );
+
+static void CC_TrackerListVPKFiles( const CCommand &args )
+{
+	const char *pchFilenameFind = ( args.ArgC() >= 2 ) ? args[1] : NULL;
+	BaseFileSystem()->m_FileTracker2.ListOpenedFiles( false, pchFilenameFind );
+}
+static ConCommand trackerlistvpkfiles( "trackerlistvpkfiles", CC_TrackerListVPKFiles, "TrackerListVPKFiles" );
+
+#endif // !DEDICATED

filesystem/filetracker.h

@@ -0,0 +1,232 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef FILETRACKER_H
+#define FILETRACKER_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+class CBaseFileSystem;
+class CPackedStoreFileHandle;
+
+#if !defined( DEDICATED )
+
+// Comments from Fletcher:
+// 
+// TF isn’t sending any hashes to the server. (That’s probably what CSGO is doing,
+// but not TF.) Comparing hashes doesn’t work when there are optional updates. (We
+// release a new client without updating the server.) Also on TF, we don’t ship
+// any textures or audio to the dedicated server.
+//  
+// On TF, the client just confirms that the files were loaded from a trusted source.
+// 
+// When a client connects to a “pure” server, the client is supposed to limit
+// which modified files are allowed.)
+
+#include "ifilelist.h"
+#include "tier1/utldict.h"
+#include "tier0/tslist.h"
+#include "tier1/stringpool.h"
+
+struct TrackedFile_t
+{
+	TrackedFile_t()
+	{
+		m_nFileFraction = 0;
+		m_PackFileID = 0;
+		m_nPackFileNumber = 0;
+		m_bPackOrVPKFile = false;
+		m_bFileInVPK = false;
+		m_iLoadedSearchPathStoreId = 0;
+		m_bIgnoredForPureServer = false;
+	}
+
+	FileHash_t m_filehashFinal;
+
+	const char *m_filename;
+	const char *m_path;
+	int m_nFileFraction;
+	int m_iLoadedSearchPathStoreId; // ID of the search path that we loaded from.  Zero if we are not currently loaded.
+
+	int m_PackFileID;
+	int m_nPackFileNumber;
+	bool m_bPackOrVPKFile;
+	bool m_bFileInVPK;
+	bool m_bIgnoredForPureServer; // Did we ignore a file by this name as a result of pure server rules, the last time it was opened?
+
+	// The crcIdentifier is a CRC32 of the filename and path. It could be used for quick comparisons of
+	//  path+filename, but since we don't do that at the moment we don't need this.
+	// CRC32_t m_crcIdentifier;
+
+	void RebuildFileName( CStringPool &stringPool, const char *pFilename, const char *pPathID, int nFileFraction );
+
+	static bool Less( const TrackedFile_t& lhs, const TrackedFile_t& rhs )
+	{
+		int nCmp = Q_strcmp( lhs.m_path, rhs.m_path );
+		if ( nCmp < 0 )
+			return true;
+		else if ( nCmp > 0 )
+			return false;
+
+		nCmp = Q_strcmp( lhs.m_filename, rhs.m_filename );
+		if ( nCmp < 0 )
+			return true;
+
+		return false;
+	}
+};
+
+struct TrackedVPKFile_t
+{
+	TrackedVPKFile_t()
+	{
+		m_PackFileID = 0;
+		m_nPackFileNumber = 0;
+		m_nFileFraction = 0;
+	}
+	int m_PackFileID;
+	int m_nPackFileNumber;
+	int m_nFileFraction;
+	int m_idxAllOpenedFiles; // Index into m_treeAllOpenedFiles
+
+	static bool Less( const TrackedVPKFile_t& lhs, const TrackedVPKFile_t& rhs )
+	{
+		if ( lhs.m_nPackFileNumber < rhs.m_nPackFileNumber )
+			return true;
+		else if ( lhs.m_nPackFileNumber > rhs.m_nPackFileNumber )
+			return false;
+
+		if ( lhs.m_nFileFraction < rhs.m_nFileFraction )
+			return true;
+		else if ( lhs.m_nFileFraction > rhs.m_nFileFraction )
+			return false;
+
+		if ( lhs.m_PackFileID < rhs.m_PackFileID )
+			return true;
+
+		return false;
+	}
+};
+
+class StuffToMD5_t
+{
+public:
+	uint8 *m_pubBuffer;
+	int m_cubBuffer;
+	MD5Value_t m_md5Value;
+    int m_idxTrackedVPKFile;
+	int m_idxListSubmittedJobs;
+};
+
+class SubmittedMd5Job_t
+{
+public:
+	bool m_bFinished;
+	MD5Value_t m_md5Value;
+};
+
+class CFileTracker2
+#ifdef SUPPORT_PACKED_STORE
+	: IThreadedFileMD5Processor
+#endif
+{
+public:
+	CFileTracker2( CBaseFileSystem *pFileSystem );
+	~CFileTracker2();
+
+	void ShutdownAsync();
+
+	void MarkAllCRCsUnverified();
+	int	GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles );
+	EFileCRCStatus CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash );
+
+#ifdef SUPPORT_PACKED_STORE
+	unsigned ThreadedProcessMD5Requests();
+	virtual int SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction );
+	virtual bool BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut );
+	virtual bool IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut );
+
+	int NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength );
+	void NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle );
+	void AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &fhandle );
+#endif
+
+	void NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId );
+	void NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength );
+	void NoteFileUnloaded( const char *pFilename, const char *pPathID );
+	int ListOpenedFiles( bool bAllOpened, const char *pchFilenameFind  );
+
+	IFileList *GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList );
+
+private:
+	int IdxFileFromName( const char *pFilename, const char *pPathID, int nFileFraction, bool bPackOrVPKFile );
+
+	CStringPool m_stringPool;
+	CUtlRBTree< TrackedFile_t, int > m_treeAllOpenedFiles;
+	CUtlRBTree< TrackedVPKFile_t, int > m_treeTrackedVPKFiles;
+
+	CBaseFileSystem					*m_pFileSystem;
+	CThreadMutex					m_Mutex;	// Threads call into here, so we need to be safe.
+
+#ifdef SUPPORT_PACKED_STORE
+	CThreadEvent m_threadEventWorkToDo;
+	CThreadEvent m_threadEventWorkCompleted;
+	volatile bool m_bThreadShouldRun;
+	ThreadHandle_t m_hWorkThread;
+
+	CTSQueue< StuffToMD5_t >				m_PendingJobs;
+	CTSQueue< StuffToMD5_t >				m_CompletedJobs;
+	CUtlLinkedList< SubmittedMd5Job_t >		m_SubmittedJobs;
+#endif // SUPPORT_PACKED_STORE
+
+	// Stats
+	int m_cThreadBlocks;
+    int m_cDupMD5s;
+};
+
+#else
+
+//
+// Dedicated server NULL filetracker. Pretty much does nothing.
+//
+class CFileTracker2
+#ifdef SUPPORT_PACKED_STORE
+	: IThreadedFileMD5Processor
+#endif
+{
+public:
+	CFileTracker2( CBaseFileSystem *pFileSystem ) {}
+	~CFileTracker2() {}
+
+	void ShutdownAsync() {}
+
+	void MarkAllCRCsUnverified() {}
+	int	GetUnverifiedFileHashes( CUnverifiedFileHash *pFiles, int nMaxFiles ) { return 0; }
+	EFileCRCStatus CheckCachedFileHash( const char *pPathID, const char *pRelativeFilename, int nFileFraction, FileHash_t *pFileHash )
+		{ return k_eFileCRCStatus_CantOpenFile; }
+
+#ifdef SUPPORT_PACKED_STORE
+	virtual int SubmitThreadedMD5Request( uint8 *pubBuffer, int cubBuffer, int PackFileID, int nPackFileNumber, int nPackFileFraction )
+		{ return 0; }
+	virtual bool BlockUntilMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) { Assert(0); return true; }
+	virtual bool IsMD5RequestComplete( int iRequest, MD5Value_t *pMd5ValueOut ) { Assert(0); return true; }
+
+	int NotePackFileOpened( const char *pVPKAbsPath, const char *pPathID, int64 nLength ) { return 0; }
+	void NotePackFileAccess( const char *pFilename, const char *pPathID, int iSearchPathStoreId, CPackedStoreFileHandle &VPKHandle ) {}
+	void AddFileHashForVPKFile( int nPackFileNumber, int nFileFraction, int cbFileLen, MD5Value_t &md5, CPackedStoreFileHandle &fhandle ) {}
+#endif
+
+	void NoteFileIgnoredForPureServer( const char *pFilename, const char *pPathID, int iSearchPathStoreId ) {}
+	void NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, int iSearchPathStoreId, FILE *fp, int64 nLength ) {}
+	void NoteFileUnloaded( const char *pFilename, const char *pPathID ) {}
+
+	IFileList *GetFilesToUnloadForWhitelistChange( IPureServerWhitelist *pNewWhiteList ) { return NULL; }
+};
+
+#endif // DEDICATED
+
+#endif // FILETRACKER_H

filesystem/linux_support.cpp

@@ -0,0 +1,266 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+//
+// $NoKeywords: $  
+//=============================================================================//
+
+#include "linux_support.h"
+#include "tier0/threadtools.h" // For ThreadInMainThread()
+#include "tier1/strtools.h"
+
+char selectBuf[PATH_MAX];
+
+int FileSelect(const struct dirent *ent)
+{
+	const char *mask=selectBuf;
+	const char *name=ent->d_name;
+
+	//printf("Test:%s %s\n",mask,name);
+
+	if(!strcmp(name,".") || !strcmp(name,"..") ) return 0;
+
+	if(!strcmp(selectBuf,"*.*")) return 1;
+
+	while( *mask && *name )
+	{
+		if(*mask=='*')
+		{
+			mask++; // move to the next char in the mask
+			if(!*mask) // if this is the end of the mask its a match 
+			{
+				return 1;
+			}
+			while(*name && toupper(*name)!=toupper(*mask)) 
+			{ // while the two don't meet up again
+				name++;
+			}
+			if(!*name) 
+			{ // end of the name
+				break; 
+			}
+		}
+		else if (*mask!='?')
+		{
+			if( toupper(*mask) != toupper(*name) )
+			{	// mismatched!
+				return 0;
+			}
+			else
+			{	
+				mask++;
+				name++;
+				if( !*mask && !*name) 
+				{ // if its at the end of the buffer
+					return 1;
+				}
+				
+			}
+
+		}
+		else /* mask is "?", we don't care*/
+		{
+			mask++;
+			name++;
+		}
+	}	
+		
+	return( !*mask && !*name ); // both of the strings are at the end
+}
+
+int FillDataStruct(FIND_DATA *dat)
+{
+	struct stat fileStat;
+
+	if(dat->curMatch >= dat->numMatches)
+		return -1;
+
+	char szFullPath[MAX_PATH];
+	Q_snprintf( szFullPath, sizeof(szFullPath), "%s/%s", dat->cBaseDir, dat->namelist[dat->curMatch]->d_name );  
+
+	if(!stat(szFullPath,&fileStat))
+	{
+		dat->dwFileAttributes=fileStat.st_mode;           
+	}
+	else
+	{
+		dat->dwFileAttributes=0;
+	}	
+
+	// now just put the filename in the output data
+	Q_snprintf( dat->cFileName, sizeof(dat->cFileName), "%s", dat->namelist[dat->curMatch]->d_name );  
+
+	//printf("%s\n", dat->namelist[dat->curMatch]->d_name);
+	free(dat->namelist[dat->curMatch]);
+
+	dat->curMatch++;
+	return 1;
+}
+
+
+HANDLE FindFirstFile( const char *fileName, FIND_DATA *dat)
+{
+	char nameStore[PATH_MAX];
+	char *dir=NULL;
+	int n,iret=-1;
+	
+	Q_strncpy(nameStore,fileName, sizeof( nameStore ) );
+
+	if(strrchr(nameStore,'/') )
+	{
+		dir=nameStore;
+		while(strrchr(dir,'/') )
+		{
+			struct stat dirChk;
+
+			// zero this with the dir name
+			dir=strrchr(nameStore,'/');
+			if ( dir == nameStore ) // special case for root dir, '/'
+			{
+				dir[1] = '\0';
+			}
+			else
+			{
+				*dir='\0';
+				dir=nameStore;
+			}
+
+			
+			if (stat(dir,&dirChk) < 0)
+			{
+				continue;
+			}
+
+			if( S_ISDIR( dirChk.st_mode ) )
+			{
+				break;	
+			}
+		}
+	}
+	else
+	{
+		// couldn't find a dir seperator...
+		return (HANDLE)-1;
+	}
+
+	if( strlen(dir)>0 )
+	{
+		if ( strlen(dir) == 1 ) // if it was the root dir
+			Q_strncpy(selectBuf,fileName+1, sizeof( selectBuf ) );
+		else
+			Q_strncpy(selectBuf,fileName+strlen(dir)+1, sizeof( selectBuf ) );
+
+		Q_strncpy(dat->cBaseDir,dir, sizeof( dat->cBaseDir ) );
+		dat->namelist = NULL;
+		n = scandir(dir, &dat->namelist, FileSelect, alphasort);
+		if (n < 0)
+		{
+			if ( dat->namelist )
+				free(dat->namelist);
+			// silently return, nothing interesting
+		}
+		else 
+		{
+			dat->numMatches = n;
+			dat->curMatch = 0;
+			iret=FillDataStruct(dat);
+			if(iret<0)
+			{
+				if ( dat->namelist )
+					free(dat->namelist);
+				dat->namelist = NULL;
+			}
+
+		}
+	}
+
+//	printf("Returning: %i \n",iret);
+	return (HANDLE)iret;
+}
+
+bool FindNextFile(HANDLE handle, FIND_DATA *dat)
+{
+	if(dat->curMatch >= dat->numMatches)
+	{	
+		if ( dat->namelist != NULL )
+			free(dat->namelist);
+		dat->namelist = NULL;
+		return false; // no matches left
+	}	
+
+	FillDataStruct(dat);
+	return true;
+}
+
+bool FindClose(HANDLE handle)
+{
+	return true;
+}
+
+
+
+// Pass this function a full path and it will look for files in the specified
+// directory that match the file name but potentially with different case.
+// The directory name itself is not treated specially.
+// If multiple names that match are found then lowercase letters take precedence.
+bool findFileInDirCaseInsensitive( const char *file, char* output, size_t bufSize)
+{
+	// Make sure the output buffer is always null-terminated.
+	output[0] = 0;
+
+	// Find where the file part starts.
+	const char *dirSep = strrchr(file,'/');
+	if( !dirSep )
+	{
+		dirSep=strrchr(file,'\\');
+		if( !dirSep ) 
+		{
+			return false;
+		}
+	}
+
+	// Allocate space for the directory portion.
+	size_t dirSize = ( dirSep - file ) + 1;
+	char *dirName = static_cast<char *>( alloca( dirSize ) ); 
+
+	V_strncpy( dirName , file, dirSize );
+
+	DIR* pDir = opendir( dirName );
+	if ( !pDir )
+		return false;
+
+	const char* filePart = dirSep + 1;
+	// The best matching file name will be placed in this array.
+	char outputFileName[ MAX_PATH ];
+	bool foundMatch = false;
+
+	// Scan through the directory.
+	for ( dirent* pEntry = NULL; ( pEntry = readdir( pDir ) ); /**/ )
+	{
+		if ( strcasecmp( pEntry->d_name, filePart ) == 0 )
+		{
+			// If we don't have an existing candidate or if this name is
+			// a better candidate then copy it in. A 'better' candidate
+			// means that test beats tesT which beats tEst -- more lowercase
+			// letters earlier equals victory.
+			if ( !foundMatch || strcmp( outputFileName, pEntry->d_name ) < 0 )
+			{
+				foundMatch = true;
+				V_strcpy_safe( outputFileName, pEntry->d_name );
+			}
+		}
+	}
+
+	closedir( pDir );
+
+	// If we didn't find any matching names then lowercase the passed in
+	// file name and use that.
+	if ( !foundMatch )
+	{
+		V_strcpy_safe( outputFileName, filePart );
+		V_strlower( outputFileName );
+	}
+
+	Q_snprintf( output, bufSize, "%s/%s", dirName, outputFileName );
+	return foundMatch;
+}

filesystem/linux_support.h

@@ -0,0 +1,58 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+// 
+// $NoKeywords: $
+//=============================================================================//
+#ifndef LINUX_SUPPORT_H
+#define LINUX_SUPPORT_H
+
+#include <ctype.h> // tolower()
+#include <limits.h> // PATH_MAX define 
+#include <string.h> //strcmp, strcpy
+#include <sys/stat.h> // stat()
+#include <unistd.h> 
+#include <dirent.h> // scandir()
+#include <stdlib.h>
+#include <stdio.h>
+#include "tier0/platform.h"
+
+#define FILE_ATTRIBUTE_DIRECTORY S_IFDIR
+
+typedef struct 
+{
+	// public data
+	int dwFileAttributes;
+	char cFileName[PATH_MAX]; // the file name returned from the call
+	char cBaseDir[PATH_MAX]; // the root dir for this find
+
+	int numMatches;
+	int curMatch;
+	struct dirent **namelist;  
+} FIND_DATA;
+
+#define WIN32_FIND_DATA FIND_DATA
+
+HANDLE FindFirstFile( const char *findName, FIND_DATA *dat);
+bool FindNextFile(HANDLE handle, FIND_DATA *dat);
+bool FindClose(HANDLE handle);
+
+// findFileInDirCaseInsensitive looks for the specified file. It returns
+// false if no directory separator is found. Otherwise it looks for the
+// requested file by scanning the specified directory and doing a case
+// insensitive match. If the file exists (in any case) then the correctly cased
+// filename will be returned in the user's buffer and 'true' will be returned.
+// If the file does not exist then the filename will be lowercased and 'false'
+// will be returned.
+// This function uses a static buffer for internal purposes and is therefore
+// not thread safe, so it must only be called from the main thread.
+bool findFileInDirCaseInsensitive( const char *file, OUT_Z_BYTECAP(bufSize) char* output, size_t bufSize );
+// The _safe version of this function should be preferred since it always infers
+// the directory size correctly.
+template<size_t bufSize>
+bool findFileInDirCaseInsensitive_safe( const char *file, OUT_Z_ARRAY char (&output)[bufSize] )
+{
+	return findFileInDirCaseInsensitive( file, output, bufSize );
+}
+
+#endif // LINUX_SUPPORT_H

filesystem/packfile.h

@@ -0,0 +1,277 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose:
+//
+// $NoKeywords: $
+//===========================================================================//
+
+#ifndef PACKFILE_H
+#define PACKFILE_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+// How many bytes compressed filehandles should hold for seeking backwards. Seeks beyond this limit will require
+// rewinding and restarting the compression, at significant performance penalty. (Warnings emitted when this occurs)
+#define PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER 4096
+
+// How many bytes compressed filehandles should attempt to read (and cache) at a time from the underlying compressed data.
+#define PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER 4096
+
+// Emit warnings in debug builds if we hold more than this many compressed file handles alive to alert to the poor
+// memory characteristics.
+#define PACKFILE_COMPRESSED_FILE_HANDLES_WARNING 20
+
+#include "basefilesystem.h"
+#include "tier1/refcount.h"
+#include "tier1/utlbuffer.h"
+#include "tier1/lzmaDecoder.h"
+
+class CPackFile;
+class CZipPackFile;
+
+// A pack file handle - essentially represents a file inside the pack file.
+class CPackFileHandle
+{
+public:
+	virtual ~CPackFileHandle() {};
+
+	virtual int Read( void* pBuffer, int nDestSize, int nBytes ) = 0;
+	virtual int Seek( int nOffset, int nWhence )                 = 0;
+	virtual int Tell()                                           = 0;
+	virtual int Size()                                           = 0;
+
+	virtual void   SetBufferSize( int nBytes ) = 0;
+	virtual int    GetSectorSize()             = 0;
+	virtual int64  AbsoluteBaseOffset()        = 0;
+};
+
+class CZipPackFileHandle : public CPackFileHandle
+{
+public:
+	CZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nLength, unsigned int nIndex = -1, unsigned int nFilePointer = 0 );
+	virtual ~CZipPackFileHandle();
+
+	virtual int Read( void* pBuffer, int nDestSize, int nBytes ) OVERRIDE;
+	virtual int Seek( int nOffset, int nWhence )                 OVERRIDE;
+
+	virtual int Tell() OVERRIDE { return m_nFilePointer; };
+	virtual int Size() OVERRIDE { return m_nLength; };
+
+	virtual void   SetBufferSize( int nBytes ) OVERRIDE;
+	virtual int    GetSectorSize()             OVERRIDE;
+	virtual int64  AbsoluteBaseOffset()        OVERRIDE;
+
+protected:
+	int64         m_nBase;        // Base offset of the file inside the pack file.
+	unsigned int  m_nFilePointer; // Current seek pointer (0 based from the beginning of the file).
+	CZipPackFile* m_pOwner;       // Pack file that owns this handle
+	unsigned int  m_nLength;      // Length of this file.
+	unsigned int  m_nIndex;       // Index into the pack's directory table
+};
+
+class CLZMAZipPackFileHandle : public CZipPackFileHandle
+{
+public:
+	CLZMAZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nOriginalSize, unsigned int nCompressedSize,
+	                        unsigned int nIndex = -1, unsigned int nFilePointer = 0 );
+	~CLZMAZipPackFileHandle();
+
+	virtual int Read( void* pBuffer, int nDestSize, int nBytes ) OVERRIDE;
+	virtual int Seek( int nOffset, int nWhence )                 OVERRIDE;
+
+	virtual int Tell() OVERRIDE;
+	virtual int Size() OVERRIDE;
+
+private:
+	// Ensure there are bytes in the read buffer, assuming we're not at the end of the underlying data
+	int FillReadBuffer();
+
+	// Reset buffers and underlying seek position to 0
+	void Reset();
+
+	// Contains the last PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER decompressed bytes. The Put and Get locations mimic our
+	// filehandle -- TellPut() == TelGet() when we are not back seeking.
+	CUtlBuffer    m_BackSeekBuffer;
+
+	// The read buffer from the underlying compressed stream. We read PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER bytes
+	// into this buffer, then consume it via the buffer get position.
+	CUtlBuffer    m_ReadBuffer;
+
+	// The decompress stream we feed our base filehandle into
+	CLZMAStream   *m_pLZMAStream;
+
+	// Current seek position in uncompressed data
+	int  m_nSeekPosition;
+
+	// Size of the decompressed data
+	unsigned int m_nOriginalSize;
+};
+
+//-----------------------------------------------------------------------------
+
+// An abstract pack file
+class CPackFile : public CRefCounted<CRefCountServiceMT>
+{
+public:
+	CPackFile();
+	virtual ~CPackFile();
+
+	// The means by which you open files:
+	virtual CFileHandle *OpenFile( const char *pFileName, const char *pOptions = "rb" ) = 0;
+
+	// Check for existance in pack
+	virtual bool ContainsFile( const char *pFileName ) = 0;
+
+	// The two functions a pack file must provide
+	virtual bool Prepare( int64 fileLen = -1, int64 nFileOfs = 0 ) = 0;
+
+	// Returns the filename for a given file in the pack. Returns true if a filename is found, otherwise buffer is filled with "unknown"
+	virtual bool IndexToFilename( int nIndex, char* buffer, int nBufferSize ) = 0;
+
+	inline int GetSectorSize();
+
+	virtual void SetupPreloadData() {}
+	virtual void DiscardPreloadData() {}
+	virtual int64 GetPackFileBaseOffset() = 0;
+
+	CBaseFileSystem *FileSystem() { return m_fs; }
+
+	// Helper for the filesystem's FindFirst/FindNext() API which mimics the old windows equivalent. pWildcard is the
+	// same pattern that you would pass to FindFirst, not a true wildcard.
+	// Mirrors the VPK code's similar call.
+	virtual void GetFileAndDirLists( const char *pFindWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput ) = 0;
+
+	// Note: threading model for pack files assumes that data
+	// is segmented into pack files that aggregate files
+	// meant to be read in one thread. Performance characteristics
+	// tuned for that case
+	CThreadFastMutex	m_mutex;
+
+	// Path management:
+	void SetPath( const CUtlSymbol &path ) { m_Path = path; }
+	const CUtlSymbol& GetPath() const	{ Assert( m_Path != UTL_INVAL_SYMBOL ); return m_Path; }
+	CUtlSymbol			m_Path;
+
+	// possibly embedded pack
+	int64				m_nBaseOffset;
+
+	CUtlString			m_ZipName;
+
+	bool				m_bIsMapPath;
+	long				m_lPackFileTime;
+
+	int					m_refCount;
+	int					m_nOpenFiles;
+
+	FILE				*m_hPackFileHandleFS;
+#if defined( SUPPORT_PACKED_STORE )
+	CPackedStoreFileHandle m_hPackFileHandleVPK;
+#endif
+	bool				m_bIsExcluded;
+
+	int					m_PackFileID;
+protected:
+	// This is the core IO routine for reading anything from a pack file, everything should go through here at some point
+	virtual int ReadFromPack( int nIndex, void* buffer, int nDestBytes, int nBytes, int64 nOffset ) = 0;
+
+	int64				m_FileLength;
+	CBaseFileSystem		*m_fs;
+
+	friend class		CPackFileHandle;
+};
+
+class CZipPackFile : public CPackFile
+{
+	friend class CZipPackFileHandle;
+public:
+	CZipPackFile( CBaseFileSystem* fs, void *pSection = NULL );
+	virtual ~CZipPackFile();
+
+	// Loads the pack file
+	virtual bool Prepare( int64 fileLen = -1, int64 nFileOfs = 0 ) OVERRIDE;
+	virtual bool ContainsFile( const char *pFileName ) OVERRIDE;
+	virtual CFileHandle *OpenFile( const char *pFileName, const char *pOptions = "rb" ) OVERRIDE;
+
+	virtual void GetFileAndDirLists( const char *pFindWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput ) OVERRIDE;
+
+	virtual int64 GetPackFileBaseOffset() OVERRIDE { return m_nBaseOffset; }
+
+	virtual bool IndexToFilename( int nIndex, char *pBuffer, int nBufferSize ) OVERRIDE;
+
+protected:
+	virtual int  ReadFromPack( int nIndex, void* buffer, int nDestBytes, int nBytes, int64 nOffset  ) OVERRIDE;
+
+	#pragma pack(1)
+
+	typedef struct
+	{
+		char name[ 112 ];
+		int64 filepos;
+		int64 filelen;
+	} packfile64_t;
+
+	typedef struct
+	{
+		char id[ 4 ];
+		int64 dirofs;
+		int64 dirlen;
+	} packheader64_t;
+
+	typedef struct
+	{
+		char id[ 8 ];
+		int64 packheaderpos;
+		int64 originalfilesize;
+	} packappenededheader_t;
+
+	#pragma pack()
+
+	// A Pack file directory entry:
+	class CPackFileEntry
+	{
+	public:
+		unsigned int		m_nPosition;
+		unsigned int		m_nOriginalSize;
+		unsigned int		m_nCompressedSize;
+		unsigned int		m_HashName;
+		unsigned short		m_nPreloadIdx;
+		unsigned short		pad;
+		unsigned short		m_nCompressionMethod;
+		FileNameHandle_t	m_hFileName;
+	};
+
+	class CPackFileLessFunc
+	{
+	public:
+		bool Less( CPackFileEntry const& src1, CPackFileEntry const& src2, void *pCtx );
+	};
+
+	// Find a file inside a pack file:
+	const CPackFileEntry* FindFile( const char* pFileName );
+
+	// Entries to the individual files stored inside the pack file.
+	CUtlSortVector< CPackFileEntry, CPackFileLessFunc > m_PackFiles;
+
+	bool						GetFileInfo( const char *pFileName, int &nBaseIndex, int64 &nFileOffset, int &nOriginalSize, int &nCompressedSize, unsigned short &nCompressionMethod );
+
+	// Preload Support
+	void						SetupPreloadData() OVERRIDE;
+	void						DiscardPreloadData() OVERRIDE;
+	ZIP_PreloadDirectoryEntry*	GetPreloadEntry( int nEntryIndex );
+
+	int64						m_nPreloadSectionOffset;
+	unsigned int				m_nPreloadSectionSize;
+	ZIP_PreloadHeader			*m_pPreloadHeader;
+	unsigned short*				m_pPreloadRemapTable;
+	ZIP_PreloadDirectoryEntry	*m_pPreloadDirectory;
+	void*						m_pPreloadData;
+	CByteswap					m_swap;
+
+#if defined ( _X360 )
+	void						*m_pSection;
+#endif
+};
+
+#endif // PACKFILE_H

filesystem/threadsaferefcountedobject.h

@@ -0,0 +1,85 @@
+//========= Copyright Valve Corporation, All rights reserved. ============//
+//
+// Purpose: 
+//
+//=============================================================================
+
+#ifndef THREADSAFEREFCOUNTEDOBJECT_H
+#define THREADSAFEREFCOUNTEDOBJECT_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+
+// This class can be used for fast access to an object from multiple threads,
+// and the main thread can wait until the threads are done using the object before it frees the object.
+template< class T >
+class CThreadSafeRefCountedObject
+{
+public:
+	CThreadSafeRefCountedObject( T initVal )
+	{
+		m_RefCount = 0;
+		m_pObject = initVal;
+		m_RefCount = 0;
+	}
+	
+	void Init( T pObj )
+	{
+		Assert( ThreadInMainThread() );
+		Assert( !m_pObject );
+		m_RefCount = 0;
+		m_pObject = pObj;
+		m_RefCount = 1;
+	}
+	
+	// Threads that access the object need to use AddRef/Release to access it.
+	T AddRef()
+	{
+		if ( ++m_RefCount > 1 )
+		{
+			return m_pObject;
+		}
+		else
+		{
+			// If the refcount was 0 when we called this, then the whitelist is about to be freed.
+			--m_RefCount;
+			return NULL;
+		}
+	}
+	void ReleaseRef( T pObj )
+	{
+		if ( --m_RefCount >= 1 )
+		{
+			Assert( m_pObject == pObj );
+		}
+	}
+
+	// The main thread can use this to access the object, since only it can Init() and Free() the object.
+	T GetInMainThread()
+	{
+		Assert( ThreadInMainThread() );
+		return m_pObject;
+	}
+
+	// The main thread calls this after it has released its last reference to the object.
+	void ResetWhenNoRemainingReferences( T newValue )
+	{
+		Assert( ThreadInMainThread() );
+		
+		// Wait until we can free it.
+		while ( m_RefCount > 0 )
+		{
+			CThread::Sleep( 20 );
+		}
+		
+		m_pObject = newValue;
+	}
+
+private:	
+	CInterlockedIntT<long>	m_RefCount;
+	T						m_pObject;
+};
+
+
+#endif // THREADSAFEREFCOUNTEDOBJECT_H

filesystem/xbox/xbox.def

@@ -0,0 +1,3 @@
+LIBRARY	filesystem_stdio_360.dll
+EXPORTS
+	CreateInterface		@1