Backups/hl2sdk
1
0
Fork 0
mirror of https://github.com/alliedmodders/hl2sdk.git synced 2025-09-19 03:56:10 +08:00
Code Activity

Sync with upstream (Issue #30).

Browse Source 
Recompiled tier1 and mathlib  for all platforms will come in next commit.
This commit is contained in:
Nicholas Hastings
2016-11-30 10:01:15 -05:00
parent 98fe5b5a34
commit 3957adff10
491 changed files with 29846 additions and 10698 deletions
Download Patch File Download Diff File Expand all files Collapse all files

343
tier1/KeyValues.cpp
View File

@ -18,14 +18,16 @@
#include <KeyValues.h>
#include "filesystem.h"
#include <vstdlib/IKeyValuesSystem.h>
#include "tier0/icommandline.h"
#include <Color.h>
#include <stdlib.h>
#include "tier0/dbg.h"
#include "tier0/mem.h"
#include "utlvector.h"
#include "utlbuffer.h"
#include "utlhash.h"
#include "utlvector.h"
#include "utlqueue.h"
#include "UtlSortVector.h"
#include "convar.h"
@ -95,16 +97,19 @@ public:
{
Assert( stackLevel >= 0 );
Assert( stackLevel < m_errorIndex );
m_errorStack[stackLevel] = symName;
if ( stackLevel < MAX_ERROR_STACK )
m_errorStack[stackLevel] = symName;
}
// Hit an error, report it and the parsing stack for context
void ReportError( const char *pError )
{
bool bSpewCR = false;
Warning( "KeyValues Error: %s in file %s\n", pError, m_pFilename );
for ( int i = 0; i < m_maxErrorIndex; i++ )
{
if ( m_errorStack[i] != INVALID_KEY_SYMBOL )
if ( i < MAX_ERROR_STACK && m_errorStack[i] != INVALID_KEY_SYMBOL )
{
if ( i < m_errorIndex )
{
@ -114,9 +119,13 @@ public:
{
Warning( "(*%s*), ", KeyValues::CallGetStringForSymbol(m_errorStack[i]) );
}
bSpewCR = true;
}
}
Warning( "\n" );
if ( bSpewCR )
Warning( "\n" );
}
private:
@ -148,6 +157,10 @@ public:
{
g_KeyValuesErrorStack.Reset( m_stackLevel, symName );
}
int GetStackLevel() const
{
return m_stackLevel;
}
private:
void Init( int symName )
{
@ -645,15 +658,62 @@ void KeyValues::UsesConditionals(bool state)
//-----------------------------------------------------------------------------
// Purpose: Load keyValues from disk
//-----------------------------------------------------------------------------
bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID )
bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool refreshCache )
{
Assert(filesystem);
#ifdef WIN32
Assert( IsX360() || ( IsPC() && _heapchk() == _HEAPOK ) );
#endif
#ifdef STAGING_ONLY
static bool s_bCacheEnabled = !!CommandLine()->FindParm( "-enable_keyvalues_cache" );
const bool bUseCache = s_bCacheEnabled && ( s_pfGetSymbolForString == KeyValues::GetSymbolForStringClassic );
#else
/*
People are cheating with the keyvalue cache enabled by doing the below, so disable it.
For example if one is to allow a blue demoman texture on sv_pure they
change it to this, "$basetexture" "temp/demoman_blue". Remember to move the
demoman texture to the temp folder in the materials folder. It will likely
not be there so make a new folder for it. Once the directory in the
demoman_blue vmt is changed to the temp folder and the vtf texture is in
the temp folder itself you are finally done.
I packed my mods into a vpk but I don't think it's required. Once in game
you must create a server via the create server button and select the map
that will load the custom texture before you join a valve server. I suggest
you only do this with player textures and such as they are always loaded.
After you load the map you join the valve server and the textures should
appear and work on valve servers.
This can be done on any sv_pure 1 server but it depends on what is type of
files are allowed. All valve servers allow temp files so that is the
example I used here."
So all vmt's files can bypass sv_pure 1. And I believe this mod is mostly
made of vmt files, so valve's sv_pure 1 bull is pretty redundant.
*/
const bool bUseCache = false;
#endif
// If pathID is null, we cannot cache the result because that has a weird iterate-through-a-bunch-of-locations behavior.
const bool bUseCacheForRead = bUseCache && !refreshCache && pathID != NULL;
const bool bUseCacheForWrite = bUseCache && pathID != NULL;
COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): Begin", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" );
// Keep a cache of keyvalues, try to load it here.
if ( bUseCacheForRead && KeyValuesSystem()->LoadFileKeyValuesFromCache( this, resourceName, pathID, filesystem ) ) {
COM_TimestampedLog( "KeyValues::LoadFromFile(%s%s%s): End / CacheHit", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "" );
return true;
}
FileHandle_t f = filesystem->Open(resourceName, "rb", pathID);
if ( !f )
{
COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / FileNotFound", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "");
return false;
}
s_LastFileLoadingFrom = (char*)resourceName;
@ -675,28 +735,41 @@ bool KeyValues::LoadFromFile( IBaseFileSystem *filesystem, const char *resourceN
buffer[fileSize+1] = 0; // double NULL terminating in case this is a unicode file
bRetOK = LoadFromBuffer( resourceName, buffer, filesystem );
}
// The cache relies on the KeyValuesSystem string table, which will only be valid if we're
// using classic mode.
if ( bUseCacheForWrite && bRetOK )
{
KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID );
}
((IFileSystem *)filesystem)->FreeOptimalReadBuffer( buffer );
( (IFileSystem *)filesystem )->FreeOptimalReadBuffer( buffer );
COM_TimestampedLog("KeyValues::LoadFromFile(%s%s%s): End / Success", pathID ? pathID : "", pathID && resourceName ? "/" : "", resourceName ? resourceName : "");
return bRetOK;
}
//-----------------------------------------------------------------------------
// Purpose: Save the keyvalues to disk
// Creates the path to the file if it doesn't exist
// Creates the path to the file if it doesn't exist
//-----------------------------------------------------------------------------
bool KeyValues::SaveToFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/ )
bool KeyValues::SaveToFile( IBaseFileSystem *filesystem, const char *resourceName, const char *pathID, bool sortKeys /*= false*/, bool bAllowEmptyString /*= false*/, bool bCacheResult /*= false*/ )
{
// create a write file
FileHandle_t f = filesystem->Open(resourceName, "wb", pathID);
if ( f == FILESYSTEM_INVALID_HANDLE )
{
DevMsg(1, "KeyValues::SaveToFile: couldn't open file \"%s\" in path \"%s\".\n",
DevMsg(1, "KeyValues::SaveToFile: couldn't open file \"%s\" in path \"%s\".\n",
resourceName?resourceName:"NULL", pathID?pathID:"NULL" );
return false;
}
KeyValuesSystem()->InvalidateCacheForFile( resourceName, pathID );
if ( bCacheResult ) {
KeyValuesSystem()->AddFileKeyValuesToCache( this, resourceName, pathID );
}
RecursiveSaveToFile(filesystem, f, NULL, 0, sortKeys, bAllowEmptyString );
filesystem->Close(f);
@ -1686,115 +1759,138 @@ void KeyValues::SetPtr( const char *keyName, void *value )
}
}
void KeyValues::RecursiveCopyKeyValues( KeyValues& src )
//-----------------------------------------------------------------------------
// Purpose: Copies the tree from the other KeyValues into this one, recursively
// beginning with the root specified by rootSrc.
//-----------------------------------------------------------------------------
void KeyValues::CopyKeyValuesFromRecursive( const KeyValues& rootSrc )
{
// garymcthack - need to check this code for possible buffer overruns.
m_iKeyName = src.GetNameSymbol();
// This code used to be recursive, which was more elegant. Unfortunately, it also blew the stack for large
// KeyValues. So now we have the iterative version which is uglier but doesn't blow the stack.
// This uses breadth-first traversal.
if( !src.m_pSub )
struct CopyStruct
{
m_iDataType = src.m_iDataType;
char buf[256];
switch( src.m_iDataType )
KeyValues* dst;
const KeyValues* src;
};
char tmp[256];
KeyValues* localDst = NULL;
CUtlQueue<CopyStruct> nodeQ;
nodeQ.Insert({ this, &rootSrc });
while ( nodeQ.Count() > 0 )
{
CopyStruct cs = nodeQ.RemoveAtHead();
// Process all the siblings of the current node. If anyone has a child, add it to the queue.
while (cs.src)
{
case TYPE_NONE:
break;
case TYPE_STRING:
if( src.m_sValue )
{
int len = Q_strlen(src.m_sValue) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, src.m_sValue, len );
}
break;
case TYPE_INT:
{
m_iValue = src.m_iValue;
Q_snprintf( buf,sizeof(buf), "%d", m_iValue );
int len = Q_strlen(buf) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, buf, len );
}
break;
case TYPE_FLOAT:
{
m_flValue = src.m_flValue;
Q_snprintf( buf,sizeof(buf), "%f", m_flValue );
int len = Q_strlen(buf) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, buf, len );
}
break;
case TYPE_PTR:
{
m_pValue = src.m_pValue;
}
break;
case TYPE_UINT64:
{
m_sValue = KVStringAlloc<char>(sizeof(uint64));
Q_memcpy( m_sValue, src.m_sValue, sizeof(uint64) );
}
break;
case TYPE_COLOR:
{
m_Color[0] = src.m_Color[0];
m_Color[1] = src.m_Color[1];
m_Color[2] = src.m_Color[2];
m_Color[3] = src.m_Color[3];
}
break;
default:
{
// do nothing . .what the heck is this?
Assert( 0 );
}
break;
}
Assert( (cs.src != NULL) == (cs.dst != NULL) );
}
#if 0
KeyValues *pDst = this;
for ( KeyValues *pSrc = src.m_pSub; pSrc; pSrc = pSrc->m_pPeer )
{
if ( pSrc->m_pSub )
{
pDst->m_pSub = new KeyValues( pSrc->m_pSub->getName() );
pDst->m_pSub->RecursiveCopyKeyValues( *pSrc->m_pSub );
}
else
{
// copy non-empty keys
if ( pSrc->m_sValue && *(pSrc->m_sValue) )
{
pDst->m_pPeer = new KeyValues(
// Copy the node contents
cs.dst->CopyKeyValue( *cs.src, sizeof(tmp), tmp );
// Add children to the queue to process later.
if (cs.src->m_pSub) {
cs.dst->m_pSub = localDst = new KeyValues( NULL );
nodeQ.Insert({ localDst, cs.src->m_pSub });
}
// Process siblings until we hit the end of the line.
if (cs.src->m_pPeer) {
cs.dst->m_pPeer = new KeyValues( NULL );
}
else {
cs.dst->m_pPeer = NULL;
}
// Advance to the next peer.
cs.src = cs.src->m_pPeer;
cs.dst = cs.dst->m_pPeer;
}
}
#endif
// Handle the immediate child
if( src.m_pSub )
{
m_pSub = new KeyValues( NULL );
m_pSub->RecursiveCopyKeyValues( *src.m_pSub );
}
// Handle the immediate peer
if( src.m_pPeer )
{
m_pPeer = new KeyValues( NULL );
m_pPeer->RecursiveCopyKeyValues( *src.m_pPeer );
}
}
KeyValues& KeyValues::operator=( KeyValues& src )
//-----------------------------------------------------------------------------
// Purpose: Copies a single KeyValue from src to this, using the provided temporary
// buffer if the keytype requires it. Does NOT recurse.
//-----------------------------------------------------------------------------
void KeyValues::CopyKeyValue( const KeyValues& src, size_t tmpBufferSizeB, char* tmpBuffer )
{
m_iKeyName = src.GetNameSymbol();
if ( src.m_pSub )
return;
m_iDataType = src.m_iDataType;
switch( src.m_iDataType )
{
case TYPE_NONE:
break;
case TYPE_STRING:
if( src.m_sValue )
{
int len = Q_strlen(src.m_sValue) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, src.m_sValue, len );
}
break;
case TYPE_INT:
{
m_iValue = src.m_iValue;
Q_snprintf( tmpBuffer, tmpBufferSizeB, "%d", m_iValue );
int len = Q_strlen(tmpBuffer) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, tmpBuffer, len );
}
break;
case TYPE_FLOAT:
{
m_flValue = src.m_flValue;
Q_snprintf( tmpBuffer, tmpBufferSizeB, "%f", m_flValue );
int len = Q_strlen(tmpBuffer) + 1;
m_sValue = KVStringAlloc<char>(len);
Q_strncpy( m_sValue, tmpBuffer, len );
}
break;
case TYPE_PTR:
{
m_pValue = src.m_pValue;
}
break;
case TYPE_UINT64:
{
m_sValue = KVStringAlloc<char>(sizeof(uint64));
Q_memcpy( m_sValue, src.m_sValue, sizeof(uint64) );
}
break;
case TYPE_COLOR:
{
m_Color[0] = src.m_Color[0];
m_Color[1] = src.m_Color[1];
m_Color[2] = src.m_Color[2];
m_Color[3] = src.m_Color[3];
}
break;
default:
{
// do nothing . .what the heck is this?
Assert( 0 );
}
break;
}
}
KeyValues& KeyValues::operator=( const KeyValues& src )
{
RemoveEverything();
Init(); // reset all values
RecursiveCopyKeyValues( src );
CopyKeyValuesFromRecursive( src );
return *this;
}
@ -1893,6 +1989,25 @@ KeyValues *KeyValues::MakeCopy( void ) const
return newKeyValue;
}
//-----------------------------------------------------------------------------
// Purpose:
//-----------------------------------------------------------------------------
KeyValues *KeyValues::MakeCopy( bool copySiblings ) const
{
KeyValues* rootDest = MakeCopy();
if ( !copySiblings )
return rootDest;
const KeyValues* curSrc = GetNextKey();
KeyValues* curDest = rootDest;
while (curSrc) {
curDest->SetNextKey( curSrc->MakeCopy() );
curDest = curDest->GetNextKey();
curSrc = curSrc->GetNextKey();
}
return rootDest;
}
//-----------------------------------------------------------------------------
// Purpose: Check if a keyName has no value assigned to it.
@ -2258,6 +2373,8 @@ bool KeyValues::LoadFromBuffer( char const *resourceName, const char *pBuffer, I
if ( !pBuffer )
return true;
COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): Begin", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : "");
int nLen = Q_strlen( pBuffer );
CUtlBuffer buf( pBuffer, nLen, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER );
@ -2270,7 +2387,11 @@ bool KeyValues::LoadFromBuffer( char const *resourceName, const char *pBuffer, I
buf.AssumeMemory( pUTF8Buf, nUTF8Len, nUTF8Len, CUtlBuffer::READ_ONLY | CUtlBuffer::TEXT_BUFFER );
}
return LoadFromBuffer( resourceName, buf, pFileSystem, pPathID );
bool retVal = LoadFromBuffer( resourceName, buf, pFileSystem, pPathID );
COM_TimestampedLog("KeyValues::LoadFromBuffer(%s%s%s): End", pPathID ? pPathID : "", pPathID && resourceName ? "/" : "", resourceName ? resourceName : "");
return retVal;
}
//-----------------------------------------------------------------------------
@ -2281,6 +2402,12 @@ void KeyValues::RecursiveLoadFromBuffer( char const *resourceName, CUtlBuffer &b
CKeyErrorContext errorReport(this);
bool wasQuoted;
bool wasConditional;
if ( errorReport.GetStackLevel() > 100 )
{
g_KeyValuesErrorStack.ReportError( "RecursiveLoadFromBuffer: recursion overflow" );
return;
}
// keep this out of the stack until a key is parsed
CKeyErrorContext errorKey( INVALID_KEY_SYMBOL );
@ -2585,7 +2712,7 @@ bool KeyValues::ReadAsBinary( CUtlBuffer &buffer, int nStackDepth )
{
char token[KEYVALUES_TOKEN_SIZE];
buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 );
buffer.GetString( token );
token[KEYVALUES_TOKEN_SIZE-1] = 0;
dat->SetName( token );
}
@ -2601,7 +2728,7 @@ bool KeyValues::ReadAsBinary( CUtlBuffer &buffer, int nStackDepth )
case TYPE_STRING:
{
char token[KEYVALUES_TOKEN_SIZE];
buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 );
buffer.GetString( token );
token[KEYVALUES_TOKEN_SIZE-1] = 0;
int len = Q_strlen( token );

2
tier1/bitbuf.cpp
View File

@ -849,7 +849,7 @@ void bf_read::SetDebugName( const char *pName )
m_pDebugName = pName;
}
void bf_read::SetOverflowFlag() RESTRICT
void bf_read::SetOverflowFlag()
{
if ( m_bAssertOnOverflow )
{

20
tier1/convar.cpp
View File

@ -592,7 +592,7 @@ void ConCommand::Dispatch( const CCommand &command )
}
// Command without callback!!!
AssertMsg( 0, ( "Encountered ConCommand '%s' without a callback!\n", GetName() ) );
AssertMsg( 0, "Encountered ConCommand '%s' without a callback!\n", GetName() );
}
@ -777,10 +777,10 @@ void ConVar::InternalSetValue( const char *value )
Q_snprintf( tempVal,sizeof(tempVal), "%f", fNewValue );
val = tempVal;
}
// Redetermine value
m_fValue = fNewValue;
m_nValue = ( int )( m_fValue );
m_nValue = ( int )( fNewValue );
if ( !( m_nFlags & FCVAR_NEVER_AS_STRING ) )
{
@ -821,13 +821,17 @@ void ConVar::ChangeStringValue( const char *tempVal, float flOldValue )
*m_pszString = 0;
}
// Invoke any necessary callback function
if ( m_fnChangeCallback )
// If nothing has changed, don't do the callbacks.
if (V_strcmp(pszOldValue, m_pszString) != 0)
{
m_fnChangeCallback( this, pszOldValue, flOldValue );
}
// Invoke any necessary callback function
if ( m_fnChangeCallback )
{
m_fnChangeCallback( this, pszOldValue, flOldValue );
}
g_pCVar->CallGlobalChangeCallbacks( this, pszOldValue, flOldValue );
g_pCVar->CallGlobalChangeCallbacks( this, pszOldValue, flOldValue );
}
stackfree( pszOldValue );
}

3
tier1/ilocalize.cpp
View File

@ -19,7 +19,8 @@
int ILocalize::ConvertANSIToUnicode(const char *ansi, wchar_t *unicode, int unicodeBufferSizeInBytes)
{
#ifdef POSIX
return Q_UTF8ToUnicode(ansi, unicode, unicodeBufferSizeInBytes);
// Q_UTF8ToUnicode returns the number of bytes. This function is expected to return the number of chars.
return Q_UTF8ToUnicode(ansi, unicode, unicodeBufferSizeInBytes) / sizeof( wchar_t );
#else
int chars = MultiByteToWideChar(CP_UTF8, 0, ansi, -1, unicode, unicodeBufferSizeInBytes / sizeof(wchar_t));
unicode[(unicodeBufferSizeInBytes / sizeof(wchar_t)) - 1] = 0;

2
tier1/interface.cpp
View File

@ -284,7 +284,7 @@ CSysModule *Sys_LoadModule( const char *pModuleName, Sys_Flags flags /* = SYS_NO
int i = CommandLine()->FindParm( "-basedir" );
if ( i )
{
strcpy( szCwd, CommandLine()->GetParm( i+1 ) );
V_strcpy_safe( szCwd, CommandLine()->GetParm( i + 1 ) );
}
}
if (szCwd[strlen(szCwd) - 1] == '/' || szCwd[strlen(szCwd) - 1] == '\\' )

29
tier1/kvpacker.cpp
View File

@ -181,7 +181,7 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer )
if ( ePackType == PACKTYPE_NULLMARKER )
break; // no more peers
buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 );
buffer.GetString( token );
token[KEYVALUES_TOKEN_SIZE-1] = 0;
dat->SetName( token );
@ -198,7 +198,7 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer )
}
case PACKTYPE_STRING:
{
buffer.GetString( token, KEYVALUES_TOKEN_SIZE-1 );
buffer.GetString( token );
token[KEYVALUES_TOKEN_SIZE-1] = 0;
dat->SetStringValue( token );
break;
@ -206,15 +206,26 @@ bool KVPacker::ReadAsBinary( KeyValues *pNode, CUtlBuffer &buffer )
case PACKTYPE_WSTRING:
{
int nLength = buffer.GetShort();
wchar_t *pTemp = (wchar_t *)stackalloc( sizeof(wchar_t) * ( 1 + nLength ) );
for( int k = 0; k < nLength; ++ k )
if ( nLength >= 0 && nLength*sizeof( uint16 ) <= (uint)buffer.GetBytesRemaining() )
{
pTemp[k] = buffer.GetShort();
}
pTemp[ nLength ] = 0;
if ( nLength > 0 )
{
wchar_t *pTemp = (wchar_t *)malloc( sizeof( wchar_t ) * (1 + nLength) );
dat->SetWString( NULL, pTemp );
for ( int k = 0; k < nLength; ++k )
{
pTemp[k] = buffer.GetShort(); // ugly, but preserving existing behavior
}
pTemp[nLength] = 0;
dat->SetWString( NULL, pTemp );
free( pTemp );
}
else
dat->SetWString( NULL, L"" );
}
break;
}

992
tier1/lzmaDecoder.cpp
View File

File diff suppressed because it is too large Load Diff

94
tier1/strtools.cpp
View File

@ -1251,7 +1251,7 @@ bool Q_RemoveAllEvilCharacters( char *pch )
int cch = Q_strlen( pch );
int cubDest = (cch + 1 ) * sizeof( wchar_t );
wchar_t *pwch = (wchar_t *)stackalloc( cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t );
bool bStrippedWhitespace = false;
@ -1289,8 +1289,13 @@ bool Q_RemoveAllEvilCharacters( char *pch )
//-----------------------------------------------------------------------------
bool Q_StripPrecedingAndTrailingWhitespaceW( wchar_t *pwch )
{
// duplicate on stack
int cch = Q_wcslen( pwch );
// Early out and don't convert if we don't have any chars or leading/trailing ws.
if ( ( cch < 1 ) || ( !iswspace( pwch[ 0 ] ) && !iswspace( pwch[ cch - 1 ] ) ) )
return false;
// duplicate on stack
int cubDest = ( cch + 1 ) * sizeof( wchar_t );
wchar_t *pwchT = (wchar_t *)stackalloc( cubDest );
Q_wcsncpy( pwchT, pwch, cubDest );
@ -1340,11 +1345,16 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespaceW( wchar_t *pwch )
//-----------------------------------------------------------------------------
bool Q_StripPrecedingAndTrailingWhitespace( char *pch )
{
// convert to unicode
int cch = Q_strlen( pch );
// Early out and don't convert if we don't have any chars or leading/trailing ws.
if ( ( cch < 1 ) || ( !isspace( (unsigned char)pch[ 0 ] ) && !isspace( (unsigned char)pch[ cch - 1 ] ) ) )
return false;
// convert to unicode
int cubDest = (cch + 1 ) * sizeof( wchar_t );
wchar_t *pwch = (wchar_t *)stackalloc( cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t );
bool bStrippedWhitespace = false;
pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, false /* not aggressive */ );
@ -1367,7 +1377,7 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespace( char *pch )
int cch = Q_strlen( pch );
int cubDest = (cch + 1 ) * sizeof( wchar_t );
wchar_t *pwch = (wchar_t *)stackalloc( cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest );
int cwch = Q_UTF8ToUnicode( pch, pwch, cubDest ) / sizeof( wchar_t );
bool bStrippedWhitespace = false;
pwch = StripWhitespaceWorker( cwch-1, pwch, &bStrippedWhitespace, true /* is aggressive */ );
@ -1381,72 +1391,10 @@ bool Q_AggressiveStripPrecedingAndTrailingWhitespace( char *pch )
return bStrippedWhitespace;
}
//-----------------------------------------------------------------------------
// Purpose: Converts a UTF8 string into a unicode string
//-----------------------------------------------------------------------------
int V_UTF8ToUnicode( const char *pUTF8, wchar_t *pwchDest, int cubDestSizeInBytes )
{
// pwchDest can be null to allow for getting the length of the string
if ( cubDestSizeInBytes > 0 )
{
AssertValidWritePtr(pwchDest);
pwchDest[0] = 0;
}
if ( !pUTF8 )
return 0;
AssertValidStringPtr(pUTF8);
#ifdef _WIN32
int cchResult = MultiByteToWideChar( CP_UTF8, 0, pUTF8, -1, pwchDest, cubDestSizeInBytes / sizeof(wchar_t) );
#elif POSIX
int cchResult = mbstowcs( pwchDest, pUTF8, cubDestSizeInBytes / sizeof(wchar_t) ) + 1;
#endif
if ( cubDestSizeInBytes > 0 )
{
pwchDest[(cubDestSizeInBytes / sizeof(wchar_t)) - 1] = 0;
}
return cchResult;
}
//-----------------------------------------------------------------------------
// Purpose: Converts a unicode string into a UTF8 (standard) string
//-----------------------------------------------------------------------------
int V_UnicodeToUTF8( const wchar_t *pUnicode, char *pUTF8, int cubDestSizeInBytes )
{
//AssertValidStringPtr(pUTF8, cubDestSizeInBytes); // no, we are sometimes pasing in NULL to fetch the length of the buffer needed.
AssertValidReadPtr(pUnicode);
if ( cubDestSizeInBytes > 0 )
{
pUTF8[0] = 0;
}
#ifdef _WIN32
int cchResult = WideCharToMultiByte( CP_UTF8, 0, pUnicode, -1, pUTF8, cubDestSizeInBytes, NULL, NULL );
#elif POSIX
int cchResult = 0;
if ( pUnicode && pUTF8 )
cchResult = wcstombs( pUTF8, pUnicode, cubDestSizeInBytes ) + 1;
#endif
if ( cubDestSizeInBytes > 0 )
{
pUTF8[cubDestSizeInBytes - 1] = 0;
}
return cchResult;
}
//-----------------------------------------------------------------------------
// Purpose: Converts a ucs2 string to a unicode (wchar_t) one, no-op on win32
//-----------------------------------------------------------------------------
int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInBytes )
int _V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInBytes )
{
Assert( cubDestSizeInBytes >= sizeof( *pUnicode ) );
AssertValidWritePtr(pUnicode);
@ -1455,7 +1403,7 @@ int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInByte
pUnicode[0] = 0;
#ifdef _WIN32
int cchResult = V_wcslen( pUCS2 );
Q_memcpy( pUnicode, pUCS2, cubDestSizeInBytes );
V_memcpy( pUnicode, pUCS2, cubDestSizeInBytes );
#else
iconv_t conv_t = iconv_open( "UCS-4LE", "UCS-2LE" );
int cchResult = -1;
@ -1486,7 +1434,7 @@ int V_UCS2ToUnicode( const ucs2 *pUCS2, wchar_t *pUnicode, int cubDestSizeInByte
//-----------------------------------------------------------------------------
// Purpose: Converts a wchar_t string into a UCS2 string -noop on windows
//-----------------------------------------------------------------------------
int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, int cubDestSizeInBytes )
int _V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, int cubDestSizeInBytes )
{
#ifdef _WIN32
// Figure out which buffer is smaller and convert from bytes to character
@ -1512,6 +1460,8 @@ int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, in
else
cchResult = cubSrcInBytes / sizeof( wchar_t );
}
#else
#error Must be implemented for this platform
#endif
return cchResult;
}
@ -1520,7 +1470,7 @@ int V_UnicodeToUCS2( const wchar_t *pUnicode, int cubSrcInBytes, char *pUCS2, in
//-----------------------------------------------------------------------------
// Purpose: Converts a ucs-2 (windows wchar_t) string into a UTF8 (standard) string
//-----------------------------------------------------------------------------
int V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes )
int _V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes )
{
AssertValidStringPtr(pUTF8, cubDestSizeInBytes);
AssertValidReadPtr(pUCS2);
@ -1574,7 +1524,7 @@ int V_UCS2ToUTF8( const ucs2 *pUCS2, char *pUTF8, int cubDestSizeInBytes )
//-----------------------------------------------------------------------------
// Purpose: Converts a UTF8 to ucs-2 (windows wchar_t)
//-----------------------------------------------------------------------------
int V_UTF8ToUCS2( const char *pUTF8, int cubSrcInBytes, ucs2 *pUCS2, int cubDestSizeInBytes )
int _V_UTF8ToUCS2( const char *pUTF8, int cubSrcInBytes, ucs2 *pUCS2, int cubDestSizeInBytes )
{
Assert( cubDestSizeInBytes >= sizeof(pUCS2[0]) );
AssertValidStringPtr(pUTF8, cubDestSizeInBytes);

572
tier1/strtools_unicode.cpp Normal file
View File

@ -0,0 +1,572 @@
//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
// $NoKeywords: $
//
//=============================================================================//
#include <limits.h>
#include "tier0/dbg.h"
#include "tier1/strtools.h"
// This code was copied from steam
#define DbgAssert Assert
//-----------------------------------------------------------------------------
// Purpose: determine if a uchar32 represents a valid Unicode code point
//-----------------------------------------------------------------------------
bool Q_IsValidUChar32( uchar32 uVal )
{
// Values > 0x10FFFF are explicitly invalid; ditto for UTF-16 surrogate halves,
// values ending in FFFE or FFFF, or values in the 0x00FDD0-0x00FDEF reserved range
return ( uVal < 0x110000u ) && ( (uVal - 0x00D800u) > 0x7FFu ) && ( (uVal & 0xFFFFu) < 0xFFFEu ) && ( ( uVal - 0x00FDD0u ) > 0x1Fu );
}
//-----------------------------------------------------------------------------
// Purpose: return number of UTF-8 bytes required to encode a Unicode code point
//-----------------------------------------------------------------------------
int Q_UChar32ToUTF8Len( uchar32 uVal )
{
DbgAssert( Q_IsValidUChar32( uVal ) );
if ( uVal <= 0x7F )
return 1;
if ( uVal <= 0x7FF )
return 2;
if ( uVal <= 0xFFFF )
return 3;
return 4;
}
//-----------------------------------------------------------------------------
// Purpose: return number of UTF-16 elements required to encode a Unicode code point
//-----------------------------------------------------------------------------
int Q_UChar32ToUTF16Len( uchar32 uVal )
{
DbgAssert( Q_IsValidUChar32( uVal ) );
if ( uVal <= 0xFFFF )
return 1;
return 2;
}
//-----------------------------------------------------------------------------
// Purpose: encode Unicode code point as UTF-8, returns number of bytes written
//-----------------------------------------------------------------------------
int Q_UChar32ToUTF8( uchar32 uVal, char *pUTF8Out )
{
DbgAssert( Q_IsValidUChar32( uVal ) );
if ( uVal <= 0x7F )
{
pUTF8Out[0] = (unsigned char) uVal;
return 1;
}
if ( uVal <= 0x7FF )
{
pUTF8Out[0] = (unsigned char)(uVal >> 6) | 0xC0;
pUTF8Out[1] = (unsigned char)(uVal & 0x3F) | 0x80;
return 2;
}
if ( uVal <= 0xFFFF )
{
pUTF8Out[0] = (unsigned char)(uVal >> 12) | 0xE0;
pUTF8Out[1] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80;
pUTF8Out[2] = (unsigned char)(uVal & 0x3F) | 0x80;
return 3;
}
pUTF8Out[0] = (unsigned char)((uVal >> 18) & 0x07) | 0xF0;
pUTF8Out[1] = (unsigned char)((uVal >> 12) & 0x3F) | 0x80;
pUTF8Out[2] = (unsigned char)((uVal >> 6) & 0x3F) | 0x80;
pUTF8Out[3] = (unsigned char)(uVal & 0x3F) | 0x80;
return 4;
}
//-----------------------------------------------------------------------------
// Purpose: encode Unicode code point as UTF-16, returns number of elements written
//-----------------------------------------------------------------------------
int Q_UChar32ToUTF16( uchar32 uVal, uchar16 *pUTF16Out )
{
DbgAssert( Q_IsValidUChar32( uVal ) );
if ( uVal <= 0xFFFF )
{
pUTF16Out[0] = (uchar16) uVal;
return 1;
}
uVal -= 0x010000;
pUTF16Out[0] = (uchar16)(uVal >> 10) | 0xD800;
pUTF16Out[1] = (uchar16)(uVal & 0x3FF) | 0xDC00;
return 2;
}
// Decode one character from a UTF-8 encoded string. Treats 6-byte CESU-8 sequences
// as a single character, as if they were a correctly-encoded 4-byte UTF-8 sequence.
int Q_UTF8ToUChar32( const char *pUTF8_, uchar32 &uValueOut, bool &bErrorOut )
{
const uint8 *pUTF8 = (const uint8 *)pUTF8_;
int nBytes = 1;
uint32 uValue = pUTF8[0];
uint32 uMinValue = 0;
// 0....... single byte
if ( uValue < 0x80 )
goto decodeFinishedNoCheck;
// Expecting at least a two-byte sequence with 0xC0 <= first <= 0xF7 (110...... and 11110...)
if ( (uValue - 0xC0u) > 0x37u || ( pUTF8[1] & 0xC0 ) != 0x80 )
goto decodeError;
uValue = (uValue << 6) - (0xC0 << 6) + pUTF8[1] - 0x80;
nBytes = 2;
uMinValue = 0x80;
// 110..... two-byte lead byte
if ( !( uValue & (0x20 << 6) ) )
goto decodeFinished;
// Expecting at least a three-byte sequence
if ( ( pUTF8[2] & 0xC0 ) != 0x80 )
goto decodeError;
uValue = (uValue << 6) - (0x20 << 12) + pUTF8[2] - 0x80;
nBytes = 3;
uMinValue = 0x800;
// 1110.... three-byte lead byte
if ( !( uValue & (0x10 << 12) ) )
goto decodeFinishedMaybeCESU8;
// Expecting a four-byte sequence, longest permissible in UTF-8
if ( ( pUTF8[3] & 0xC0 ) != 0x80 )
goto decodeError;
uValue = (uValue << 6) - (0x10 << 18) + pUTF8[3] - 0x80;
nBytes = 4;
uMinValue = 0x10000;
// 11110... four-byte lead byte. fall through to finished.
decodeFinished:
if ( uValue >= uMinValue && Q_IsValidUChar32( uValue ) )
{
decodeFinishedNoCheck:
uValueOut = uValue;
bErrorOut = false;
return nBytes;
}
decodeError:
uValueOut = '?';
bErrorOut = true;
return nBytes;
decodeFinishedMaybeCESU8:
// Do we have a full UTF-16 surrogate pair that's been UTF-8 encoded afterwards?
// That is, do we have 0xD800-0xDBFF followed by 0xDC00-0xDFFF? If so, decode it all.
if ( ( uValue - 0xD800u ) < 0x400u && pUTF8[3] == 0xED && (uint8)( pUTF8[4] - 0xB0 ) < 0x10 && ( pUTF8[5] & 0xC0 ) == 0x80 )
{
uValue = 0x10000 + ( ( uValue - 0xD800u ) << 10 ) + ( (uint8)( pUTF8[4] - 0xB0 ) << 6 ) + pUTF8[5] - 0x80;
nBytes = 6;
uMinValue = 0x10000;
}
goto decodeFinished;
}
// Decode one character from a UTF-16 encoded string.
int Q_UTF16ToUChar32( const uchar16 *pUTF16, uchar32 &uValueOut, bool &bErrorOut )
{
if ( Q_IsValidUChar32( pUTF16[0] ) )
{
uValueOut = pUTF16[0];
bErrorOut = false;
return 1;
}
else if ( (pUTF16[0] - 0xD800u) < 0x400u && (pUTF16[1] - 0xDC00u) < 0x400u )
{
// Valid surrogate pair, but maybe not encoding a valid Unicode code point...
uchar32 uVal = 0x010000 + ((pUTF16[0] - 0xD800u) << 10) + (pUTF16[1] - 0xDC00);
if ( Q_IsValidUChar32( uVal ) )
{
uValueOut = uVal;
bErrorOut = false;
return 2;
}
else
{
uValueOut = '?';
bErrorOut = true;
return 2;
}
}
else
{
uValueOut = '?';
bErrorOut = true;
return 1;
}
}
namespace // internal use only
{
// Identity transformations and validity tests for use with Q_UnicodeConvertT
int Q_UTF32ToUChar32( const uchar32 *pUTF32, uchar32 &uVal, bool &bErr )
{
bErr = !Q_IsValidUChar32( *pUTF32 );
uVal = bErr ? '?' : *pUTF32;
return 1;
}
int Q_UChar32ToUTF32Len( uchar32 uVal )
{
return 1;
}
int Q_UChar32ToUTF32( uchar32 uVal, uchar32 *pUTF32 )
{
*pUTF32 = uVal;
return 1;
}
// A generic Unicode processing loop: decode one character from input to uchar32, handle errors, encode uchar32 to output
template < typename SrcType, typename DstType, bool bStopAtNull, int (&DecodeSrc)( const SrcType*, uchar32&, bool& ), int (&EncodeDstLen)( uchar32 ), int (&EncodeDst)( uchar32, DstType* ) >
int Q_UnicodeConvertT( const SrcType *pIn, int nInChars, DstType *pOut, int nOutBytes, EStringConvertErrorPolicy ePolicy )
{
if ( !pIn )
{
// For now, assert and return 0. Once these are cleaned out a bit
// we should remove this return and just leave in the assert...
AssertMsg( pIn, "We shouldn't be passing in NULL!" );
return 0;
}
int nOut = 0;
if ( !pOut )
{
while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) )
{
uchar32 uVal;
// Initialize in order to avoid /analyze warnings.
bool bErr = false;
pIn += DecodeSrc( pIn, uVal, bErr );
nOut += EncodeDstLen( uVal );
if ( bErr )
{
#ifdef _DEBUG
AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" );
#endif
if ( ePolicy & _STRINGCONVERTFLAG_SKIP )
{
nOut -= EncodeDstLen( uVal );
}
else if ( ePolicy & _STRINGCONVERTFLAG_FAIL )
{
pOut[0] = 0;
return 0;
}
}
}
}
else
{
int nOutElems = nOutBytes / sizeof( DstType );
if ( nOutElems <= 0 )
return 0;
int nMaxOut = nOutElems - 1;
while ( bStopAtNull ? ( *pIn ) : ( nInChars-- > 0 ) )
{
uchar32 uVal;
// Initialize in order to avoid /analyze warnings.
bool bErr = false;
pIn += DecodeSrc( pIn, uVal, bErr );
if ( nOut + EncodeDstLen( uVal ) > nMaxOut )
break;
nOut += EncodeDst( uVal, pOut + nOut );
if ( bErr )
{
#ifdef _DEBUG
AssertMsg( !(ePolicy & _STRINGCONVERTFLAG_ASSERT), "invalid Unicode byte sequence" );
#endif
if ( ePolicy & _STRINGCONVERTFLAG_SKIP )
{
nOut -= EncodeDstLen( uVal );
}
else if ( ePolicy & _STRINGCONVERTFLAG_FAIL )
{
pOut[0] = 0;
return 0;
}
}
}
pOut[nOut] = 0;
}
return (nOut + 1) * sizeof( DstType );
}
}
//-----------------------------------------------------------------------------
// Purpose: Returns true if UTF-8 string contains invalid sequences.
//-----------------------------------------------------------------------------
bool Q_UnicodeValidate( const char *pUTF8 )
{
bool bError = false;
while ( *pUTF8 )
{
uchar32 uVal;
// Our UTF-8 decoder silently fixes up 6-byte CESU-8 (improperly re-encoded UTF-16) sequences.
// However, these are technically not valid UTF-8. So if we eat 6 bytes at once, it's an error.
int nCharSize = Q_UTF8ToUChar32( pUTF8, uVal, bError );
if ( bError || nCharSize == 6 )
return false;
pUTF8 += nCharSize;
}
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Returns true if UTF-16 string contains invalid sequences.
//-----------------------------------------------------------------------------
bool Q_UnicodeValidate( const uchar16 *pUTF16 )
{
bool bError = false;
while ( *pUTF16 )
{
uchar32 uVal;
pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError );
if ( bError )
return false;
}
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Returns true if UTF-32 string contains invalid sequences.
//-----------------------------------------------------------------------------
bool Q_UnicodeValidate( const uchar32 *pUTF32 )
{
while ( *pUTF32 )
{
if ( !Q_IsValidUChar32( *pUTF32++ ) )
return false;
++pUTF32;
}
return true;
}
//-----------------------------------------------------------------------------
// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-8 string
//-----------------------------------------------------------------------------
int Q_UnicodeLength( const char *pUTF8 )
{
int nChars = 0;
while ( *pUTF8 )
{
bool bError;
uchar32 uVal;
pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError );
++nChars;
}
return nChars;
}
//-----------------------------------------------------------------------------
// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-16 string
//-----------------------------------------------------------------------------
int Q_UnicodeLength( const uchar16 *pUTF16 )
{
int nChars = 0;
while ( *pUTF16 )
{
bool bError;
uchar32 uVal;
pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError );
++nChars;
}
return nChars;
}
//-----------------------------------------------------------------------------
// Purpose: Returns number of Unicode code points (aka glyphs / characters) encoded in the UTF-32 string
//-----------------------------------------------------------------------------
int Q_UnicodeLength( const uchar32 *pUTF32 )
{
int nChars = 0;
while ( *pUTF32++ )
++nChars;
return nChars;
}
//-----------------------------------------------------------------------------
// Purpose: Advance a UTF-8 string pointer by a certain number of Unicode code points, stopping at end of string
//-----------------------------------------------------------------------------
char *Q_UnicodeAdvance( char *pUTF8, int nChars )
{
while ( nChars > 0 && *pUTF8 )
{
uchar32 uVal;
bool bError;
pUTF8 += Q_UTF8ToUChar32( pUTF8, uVal, bError );
--nChars;
}
return pUTF8;
}
//-----------------------------------------------------------------------------
// Purpose: Advance a UTF-16 string pointer by a certain number of Unicode code points, stopping at end of string
//-----------------------------------------------------------------------------
uchar16 *Q_UnicodeAdvance( uchar16 *pUTF16, int nChars )
{
while ( nChars > 0 && *pUTF16 )
{
uchar32 uVal;
bool bError;
pUTF16 += Q_UTF16ToUChar32( pUTF16, uVal, bError );
--nChars;
}
return pUTF16;
}
//-----------------------------------------------------------------------------
// Purpose: Advance a UTF-32 string pointer by a certain number of Unicode code points, stopping at end of string
//-----------------------------------------------------------------------------
uchar32 *Q_UnicodeAdvance( uchar32 *pUTF32, int nChars )
{
while ( nChars > 0 && *pUTF32 )
{
++pUTF32;
--nChars;
}
return pUTF32;
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF8ToUTF16( const char *pUTF8, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< char, uchar16, true, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, 0, pUTF16, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF8ToUTF32( const char *pUTF8, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< char, uchar32, true, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, 0, pUTF32, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF16ToUTF8( const uchar16 *pUTF16, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar16, char, true, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, 0, pUTF8, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF16ToUTF32( const uchar16 *pUTF16, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar16, uchar32, true, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, 0, pUTF32, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF32ToUTF8( const uchar32 *pUTF32, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, char, true, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, 0, pUTF8, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF32ToUTF16( const uchar32 *pUTF32, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, uchar16, true, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, 0, pUTF16, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF32ToUTF32( const uchar32 *pUTF32Source, uchar32 *pUTF32Dest, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32Source, 0, pUTF32Dest, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF8CharsToUTF16( const char *pUTF8, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< char, uchar16, false, Q_UTF8ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF8, nElements, pUTF16, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF8CharsToUTF32( const char *pUTF8, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< char, uchar32, false, Q_UTF8ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF8, nElements, pUTF32, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF16CharsToUTF8( const uchar16 *pUTF16, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar16, char, false, Q_UTF16ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF16, nElements, pUTF8, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF16CharsToUTF32( const uchar16 *pUTF16, int nElements, uchar32 *pUTF32, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar16, uchar32, false, Q_UTF16ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF16, nElements, pUTF32, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF32CharsToUTF8( const uchar32 *pUTF32, int nElements, char *pUTF8, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, char, false, Q_UTF32ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF32, nElements, pUTF8, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Perform conversion. Returns number of *bytes* required if output pointer is NULL.
//-----------------------------------------------------------------------------
int Q_UTF32CharsToUTF16( const uchar32 *pUTF32, int nElements, uchar16 *pUTF16, int cubDestSizeInBytes, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, uchar16, false, Q_UTF32ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF32, nElements, pUTF16, cubDestSizeInBytes, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Repair a UTF-8 string by removing or replacing invalid seqeuences. Returns non-zero on success.
//-----------------------------------------------------------------------------
int Q_UnicodeRepair( char *pUTF8, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< char, char, true, Q_UTF8ToUChar32, Q_UChar32ToUTF8Len, Q_UChar32ToUTF8 >( pUTF8, 0, pUTF8, INT_MAX, ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Repair a UTF-16 string by removing or replacing invalid seqeuences. Returns non-zero on success.
//-----------------------------------------------------------------------------
int Q_UnicodeRepair( uchar16 *pUTF16, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar16, uchar16, true, Q_UTF16ToUChar32, Q_UChar32ToUTF16Len, Q_UChar32ToUTF16 >( pUTF16, 0, pUTF16, INT_MAX/sizeof(uchar16), ePolicy );
}
//-----------------------------------------------------------------------------
// Purpose: Repair a UTF-32 string by removing or replacing invalid seqeuences. Returns non-zero on success.
//-----------------------------------------------------------------------------
int Q_UnicodeRepair( uchar32 *pUTF32, EStringConvertErrorPolicy ePolicy )
{
return Q_UnicodeConvertT< uchar32, uchar32, true, Q_UTF32ToUChar32, Q_UChar32ToUTF32Len, Q_UChar32ToUTF32 >( pUTF32, 0, pUTF32, INT_MAX/sizeof(uchar32), ePolicy );
}

10
tier1/tier1.vpc
View File

@ -64,6 +64,7 @@ $Project "tier1"
$File "reliabletimer.cpp"
$File "stringpool.cpp"
$File "strtools.cpp"
$File "strtools_unicode.cpp"
$File "tier1.cpp"
$File "tokenreader.cpp"
$File "sparsematrix.cpp"
@ -79,6 +80,15 @@ $Project "tier1"
$File "snappy-stubs-internal.cpp"
}
// Select bits from the LZMA SDK to support lzmaDecoder.h
// Encoding support requires the full lzma project
$Folder "LZMA Decompression Support"
{
$File "$SRCDIR\utils\lzma\C\LzmaDec.h"
$File "$SRCDIR\utils\lzma\C\LzmaDec.c"
$File "$SRCDIR\utils\lzma\C\7zTypes.h"
}
$Folder "Header Files"
{
$Folder "Internal Header Files"

2
tier1/uniqueid.cpp
View File

@ -163,7 +163,7 @@ bool Unserialize( CUtlBuffer &buf, UniqueId_t &dest )
{
int nTextLen = buf.PeekStringLength();
char *pBuf = (char*)stackalloc( nTextLen );
buf.GetString( pBuf, nTextLen );
buf.GetStringManualCharCount( pBuf, nTextLen );
UniqueIdFromString( &dest, pBuf, nTextLen );
}
else

36
tier1/utlbuffer.cpp
View File

@ -609,17 +609,19 @@ int CUtlBuffer::PeekDelimitedStringLength( CUtlCharConversion *pConv, bool bActu
//-----------------------------------------------------------------------------
// Reads a null-terminated string
//-----------------------------------------------------------------------------
void CUtlBuffer::GetString( char* pString, int nMaxChars )
void CUtlBuffer::GetStringInternal( char *pString, size_t maxLenInChars )
{
if (!IsValid())
if ( !IsValid() )
{
*pString = 0;
return;
}
if ( nMaxChars == 0 )
Assert( maxLenInChars != 0 );
if ( maxLenInChars == 0 )
{
nMaxChars = INT_MAX;
return;
}
// Remember, this *includes* the null character
@ -631,24 +633,21 @@ void CUtlBuffer::GetString( char* pString, int nMaxChars )
EatWhiteSpace();
}
if ( nLen == 0 )
if ( nLen <= 0 )
{
*pString = 0;
m_Error |= GET_OVERFLOW;
return;
}
// Strip off the terminating NULL
if ( nLen <= nMaxChars )
const size_t nCharsToRead = min( (size_t)nLen, maxLenInChars ) - 1;
Get( pString, nCharsToRead );
pString[nCharsToRead] = 0;
if ( (size_t)nLen > (nCharsToRead + 1) )
{
Get( pString, nLen - 1 );
pString[ nLen - 1 ] = 0;
}
else
{
Get( pString, nMaxChars - 1 );
pString[ nMaxChars - 1 ] = 0;
SeekGet( SEEK_CURRENT, nLen - 1 - nMaxChars );
SeekGet( SEEK_CURRENT, nLen - (nCharsToRead + 1) );
}
// Read the terminating NULL in binary formats
@ -733,7 +732,7 @@ void CUtlBuffer::GetDelimitedString( CUtlCharConversion *pConv, char *pString, i
{
if ( !IsText() || !pConv )
{
GetString( pString, nMaxChars );
GetStringInternal( pString, nMaxChars );
return;
}
@ -1045,7 +1044,7 @@ int CUtlBuffer::VaScanf( const char* pFmt, va_list list )
case 's':
{
char* s = va_arg( list, char * );
GetString( s );
GetStringInternal( s, 256 );
}
break;
@ -1736,7 +1735,6 @@ void CUtlBuffer::Swap( CUtlMemory<uint8> &mem )
CUtlInplaceBuffer::CUtlInplaceBuffer( int growSize /* = 0 */, int initSize /* = 0 */, int nFlags /* = 0 */ ) :
CUtlBuffer( growSize, initSize, nFlags )
{
NULL;
}
bool CUtlInplaceBuffer::InplaceGetLinePtr( char **ppszInBufferPtr, int *pnLineLength )