Backups/UnityExplorer_Fix
1
0
Fork 0
mirror of https://github.com/GrahamKracker/UnityExplorer.git synced 2025-07-01 11:12:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rewrite lexer from scratch

Browse Source
This commit is contained in:
Sinai
2021-05-10 15:58:49 +10:00
parent 06122fe8c9
commit caad39bb9a
20 changed files with 650 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
src/UI/CSConsole/CSConsole.cs
View File

@ -55,7 +55,7 @@ The following helper methods are available:
#endregion
public static ScriptEvaluator Evaluator;
public static CSLexer Lexer;
public static LexerBuilder Lexer;
private static StringBuilder evaluatorOutput;
private static HashSet<string> usingDirectives;
@ -72,7 +72,7 @@ The following helper methods are available:
{
try
{
Lexer = new CSLexer();
Lexer = new LexerBuilder();
ResetConsole(false);
Evaluator.Compile("0 == 0");
@ -173,8 +173,8 @@ The following helper methods are available:
}
}
if (EnableAutoIndent && InputManager.GetKeyDown(KeyCode.Return))
DoAutoIndent();
//if (EnableAutoIndent && InputManager.GetKeyDown(KeyCode.Return))
// DoAutoIndent();
//if (EnableAutocompletes && InputField.isFocused)
//{
@ -183,55 +183,31 @@ The following helper methods are available:
//}
}
// Invoked at most once per frame
private static void OnConsoleInputChanged(string input)
{
// todo update auto completes
// todo update syntax highlight
// update syntax highlight
Panel.HighlightText.text = Lexer.SyntaxHighlight(input);
}
// TODO?
//private static void DoAutoIndent()
//{
// int caret = Panel.LastCaretPosition;
// Panel.InputField.Text = Lexer.AutoIndentOnEnter(InputField.text, ref caret);
// InputField.caretPosition = caret;
//
// Panel.InputText.Rebuild(CanvasUpdate.Prelayout);
// InputField.ForceLabelUpdate();
// InputField.Rebuild(CanvasUpdate.Prelayout);
//
// OnConsoleInputChanged(InputField.text);
//}
// Autocompletes
private static string SyntaxHighlight(string input)
{
var sb = new StringBuilder();
int curIdx = 0;
foreach (var match in Lexer.GetMatches(input))
{
// append non-highlighted text between last match and this
for (int i = curIdx; i < match.startIndex; i++)
sb.Append(input[i]);
// append the highlighted match
sb.Append(match.htmlColorTag);
for (int i = match.startIndex; i < match.endIndex; i++)
sb.Append(input[i]);
sb.Append(SignatureHighlighter.CLOSE_COLOR);
// update the index
curIdx = match.endIndex;
}
return sb.ToString();
}
// Indent
private static void DoAutoIndent()
{
int caret = Panel.LastCaretPosition;
Panel.InputField.Text = Lexer.AutoIndentOnEnter(InputField.text, ref caret);
InputField.caretPosition = caret;
Panel.InputText.Rebuild(CanvasUpdate.Prelayout);
InputField.ForceLabelUpdate();
InputField.Rebuild(CanvasUpdate.Prelayout);
OnConsoleInputChanged(InputField.text);
}
}
}

355
src/UI/CSConsole/CSLexer.cs
View File

@ -1,355 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine.UI;
using UnityExplorer.UI.CSharpConsole.Lexer;
namespace UnityExplorer.UI.CSharpConsole
{
public struct LexerMatchInfo
{
public int startIndex;
public int endIndex;
public string htmlColorTag;
}
public enum DelimiterType
{
Start,
End,
};
public class CSLexer
{
private string inputString;
private readonly Matcher[] matchers;
private readonly HashSet<char> startDelimiters;
private readonly HashSet<char> endDelimiters;
private int currentIndex;
private int currentLookaheadIndex;
public char Current { get; private set; }
public char Previous { get; private set; }
public bool EndOfStream => currentLookaheadIndex >= inputString.Length;
public static char indentOpen = '{';
public static char indentClose = '}';
//private static StringBuilder indentBuilder = new StringBuilder();
public static char[] delimiters = new[]
{
'[', ']', '(', ')', '{', '}', ';', ':', ',', '.'
};
private static readonly CommentMatch commentMatcher = new CommentMatch();
private static readonly SymbolMatch symbolMatcher = new SymbolMatch();
private static readonly NumberMatch numberMatcher = new NumberMatch();
private static readonly StringMatch stringMatcher = new StringMatch();
private static readonly KeywordMatch keywordMatcher = new KeywordMatch();
public CSLexer()
{
startDelimiters = new HashSet<char>(delimiters);
endDelimiters = new HashSet<char>(delimiters);
this.matchers = new Matcher[]
{
commentMatcher,
symbolMatcher,
numberMatcher,
stringMatcher,
keywordMatcher,
};
foreach (Matcher lexer in matchers)
{
foreach (char c in lexer.StartChars)
{
if (!startDelimiters.Contains(c))
startDelimiters.Add(c);
}
foreach (char c in lexer.EndChars)
{
if (!endDelimiters.Contains(c))
endDelimiters.Add(c);
}
}
}
public IEnumerable<LexerMatchInfo> GetMatches(string input)
{
if (input == null || matchers == null || matchers.Length == 0)
yield break;
inputString = input;
Current = ' ';
Previous = ' ';
currentIndex = 0;
currentLookaheadIndex = 0;
while (!EndOfStream)
{
bool didMatchLexer = false;
ReadWhiteSpace();
foreach (Matcher matcher in matchers)
{
int startIndex = currentIndex;
bool isMatched = matcher.IsMatch(this);
if (isMatched)
{
int endIndex = currentIndex;
didMatchLexer = true;
yield return new LexerMatchInfo
{
startIndex = startIndex,
endIndex = endIndex,
htmlColorTag = matcher.HexColorTag,
};
break;
}
}
if (!didMatchLexer)
{
ReadNext();
Commit();
}
}
}
// Lexer reading
public char ReadNext()
{
if (EndOfStream)
return '\0';
Previous = Current;
Current = inputString[currentLookaheadIndex];
currentLookaheadIndex++;
return Current;
}
public void Rollback(int amount = -1)
{
if (amount == -1)
currentLookaheadIndex = currentIndex;
else
{
if (currentLookaheadIndex > currentIndex)
currentLookaheadIndex -= amount;
}
int previousIndex = currentLookaheadIndex - 1;
if (previousIndex >= inputString.Length)
Previous = inputString[inputString.Length - 1];
else if (previousIndex >= 0)
Previous = inputString[previousIndex];
else
Previous = ' ';
}
public void Commit()
{
currentIndex = currentLookaheadIndex;
}
public bool IsSpecialSymbol(char character, DelimiterType position = DelimiterType.Start)
{
if (position == DelimiterType.Start)
return startDelimiters.Contains(character);
return endDelimiters.Contains(character);
}
private void ReadWhiteSpace()
{
while (char.IsWhiteSpace(ReadNext()) == true)
Commit();
Rollback();
}
// Auto-indenting
public int GetIndentLevel(string input, int toIndex)
{
bool stringState = false;
int indent = 0;
for (int i = 0; i < toIndex && i < input.Length; i++)
{
char character = input[i];
if (character == '"')
stringState = !stringState;
else if (!stringState && character == indentOpen)
indent++;
else if (!stringState && character == indentClose)
indent--;
}
if (indent < 0)
indent = 0;
return indent;
}
// TODO not quite correct, but almost there.
public string AutoIndentOnEnter(string input, ref int caretPos)
{
var sb = new StringBuilder(input);
bool inString = false;
bool inChar = false;
int currentIndent = 0;
int curLineIndent = 0;
bool prevWasNewLine = true;
// process before caret position
for (int i = 0; i < caretPos; i++)
{
char c = sb[i];
ExplorerCore.Log(i + ": " + c);
// update string/char state
if (!inChar && c == '\"')
inString = !inString;
else if (!inString && c == '\'')
inChar = !inChar;
// continue if inside string or char
if (inString || inChar)
continue;
// check for new line
if (c == '\n')
{
ExplorerCore.Log("new line, resetting line counts");
curLineIndent = 0;
prevWasNewLine = true;
}
// check for indent
else if (c == '\t' && prevWasNewLine)
{
ExplorerCore.Log("its a tab");
if (curLineIndent > currentIndent)
{
ExplorerCore.Log("too many tabs, removing");
// already reached the indent we should have
sb.Remove(i, 1);
i--;
caretPos--;
curLineIndent--;
}
else
curLineIndent++;
}
// remove spaces on new lines
else if (c == ' ' && prevWasNewLine)
{
ExplorerCore.Log("removing newline-space");
sb.Remove(i, 1);
i--;
caretPos--;
}
else
{
if (c == indentClose)
currentIndent--;
if (prevWasNewLine && curLineIndent < currentIndent)
{
ExplorerCore.Log("line is not indented enough");
// line is not indented enough
int diff = currentIndent - curLineIndent;
sb.Insert(i, new string('\t', diff));
caretPos += diff;
i += diff;
}
// check for brackets
if ((c == indentClose || c == indentOpen) && !prevWasNewLine)
{
ExplorerCore.Log("bracket needs new line");
// need to put it on a new line
sb.Insert(i, $"\n{new string('\t', currentIndent)}");
caretPos += 1 + currentIndent;
i += 1 + currentIndent;
}
if (c == indentOpen)
currentIndent++;
prevWasNewLine = false;
}
}
// todo put caret on new line after previous bracket if needed
// indent caret to current indent
// process after caret position, make sure there are equal opened/closed brackets
ExplorerCore.Log("-- after caret --");
for (int i = caretPos; i < sb.Length; i++)
{
char c = sb[i];
ExplorerCore.Log(i + ": " + c);
// update string/char state
if (!inChar && c == '\"')
inString = !inString;
else if (!inString && c == '\'')
inChar = !inChar;
if (inString || inChar)
continue;
if (c == indentOpen)
currentIndent++;
else if (c == indentClose)
currentIndent--;
}
if (currentIndent > 0)
{
ExplorerCore.Log("there are not enough closing brackets, curIndent is " + currentIndent);
// There are not enough close brackets
// TODO this should append in reverse indent order (small indents inserted first, then biggest).
while (currentIndent > 0)
{
ExplorerCore.Log("Inserting closing bracket with " + currentIndent + " indent");
// append the indented '}' on a new line
sb.Insert(caretPos, $"\n{new string('\t', currentIndent - 1)}}}");
currentIndent--;
}
}
//else if (currentIndent < 0)
//{
// // There are too many close brackets
//
// // todo?
//}
return sb.ToString();
}
}
}

52
src/UI/CSConsole/Lexer/CommentLexer.cs Normal file
View File

@ -0,0 +1,52 @@
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public class CommentLexer : Lexer
{
private enum CommentType
{
Line,
Block
}
protected override Color HighlightColor => new Color(0.34f, 0.65f, 0.29f, 1.0f);
public override bool TryMatchCurrent(LexerBuilder lexer)
{
if (lexer.Current == '/')
{
lexer.PeekNext();
if (lexer.Current == '/')
{
// line comment. read to end of line or file.
do
{
lexer.Commit();
lexer.PeekNext();
}
while (!lexer.EndOrNewLine);
return true;
}
else if (lexer.Current == '*')
{
// block comment, read until end of file or closing '*/'
lexer.PeekNext();
do
{
lexer.PeekNext();
lexer.Commit();
}
while (!lexer.EndOfInput && !(lexer.Current == '/' && lexer.Previous == '*'));
return true;
}
}
return false;
}
}
}

48
src/UI/CSConsole/Lexer/CommentMatch.cs
View File

@ -1,48 +0,0 @@
using System.Collections.Generic;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public class CommentMatch : Matcher
{
public string lineCommentStart = @"//";
public string blockCommentStart = @"/*";
public string blockCommentEnd = @"*/";
public override Color HighlightColor => new Color(0.34f, 0.65f, 0.29f, 1.0f);
public override IEnumerable<char> StartChars => new char[] { lineCommentStart[0], blockCommentStart[0] };
public override IEnumerable<char> EndChars => new char[] { blockCommentEnd[0] };
public override bool IsImplicitMatch(CSLexer lexer) => IsMatch(lexer, lineCommentStart) || IsMatch(lexer, blockCommentStart);
private bool IsMatch(CSLexer lexer, string commentType)
{
if (!string.IsNullOrEmpty(commentType))
{
lexer.Rollback();
bool match = true;
for (int i = 0; i < commentType.Length; i++)
{
if (commentType[i] != lexer.ReadNext())
{
match = false;
break;
}
}
if (match)
{
// Read until end of line or file
while (!IsEndLineOrEndFile(lexer, lexer.ReadNext())) { }
return true;
}
}
return false;
}
private bool IsEndLineOrEndFile(CSLexer lexer, char character) => lexer.EndOfStream || character == '\n' || character == '\r';
}
}

61
src/UI/CSConsole/Lexer/KeywordLexer.cs Normal file
View File

@ -0,0 +1,61 @@
using System.Collections.Generic;
using System.Text;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public class KeywordLexer : Lexer
{
private readonly string[] Keywords = new[] { "add", "as", "ascending", "await", "bool", "break", "by", "byte",
"case", "catch", "char", "checked", "const", "continue", "decimal", "default", "descending", "do", "dynamic",
"else", "equals", "false", "finally", "float", "for", "foreach", "from", "global", "goto", "group", "if", "in",
"int", "into", "is", "join", "let", "lock", "long", "new", "null", "object", "on", "orderby", "out", "ref",
"remove", "return", "sbyte", "select", "short", "sizeof", "stackalloc", "string", "switch", "throw", "true",
"try", "typeof", "uint", "ulong", "ushort", "var", "where", "while", "yield", "abstract", "async", "base",
"class", "delegate", "enum", "explicit", "extern", "fixed", "get", "implicit", "interface", "internal",
"namespace", "operator", "override", "params", "private", "protected", "public", "using", "partial", "readonly",
"sealed", "set", "static", "struct", "this", "unchecked", "unsafe", "value", "virtual", "volatile", "void" };
private readonly Dictionary<int, HashSet<string>> keywordsByLength = new Dictionary<int, HashSet<string>>();
public KeywordLexer()
{
foreach (var kw in Keywords)
{
if (!keywordsByLength.ContainsKey(kw.Length))
keywordsByLength.Add(kw.Length, new HashSet<string>());
keywordsByLength[kw.Length].Add(kw);
}
}
protected override Color HighlightColor => new Color(0.33f, 0.61f, 0.83f, 1.0f);
public override bool TryMatchCurrent(LexerBuilder lexer)
{
if (!lexer.IsDelimiter(lexer.Previous, true))
return false;
int len = 0;
var sb = new StringBuilder();
while (!lexer.EndOfInput)
{
sb.Append(lexer.Current);
len++;
var next = lexer.PeekNext();
if (lexer.IsDelimiter(next, true))
{
lexer.RollbackBy(1);
break;
}
}
if (keywordsByLength.TryGetValue(len, out var keywords) && keywords.Contains(sb.ToString()))
{
lexer.Commit();
return true;
}
return false;
}
}
}

91
src/UI/CSConsole/Lexer/KeywordMatch.cs
View File

@ -1,91 +0,0 @@
using System.Collections.Generic;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public class KeywordMatch : Matcher
{
public string[] Keywords = new[] { "add", "as", "ascending", "await", "bool", "break", "by", "byte",
"case", "catch", "char", "checked", "const", "continue", "decimal", "default", "descending", "do", "dynamic",
"else", "equals", "false", "finally", "float", "for", "foreach", "from", "global", "goto", "group",
"if", "in", "int", "into", "is", "join", "let", "lock", "long", "new", "null", "object", "on", "orderby", "out",
"ref", "remove", "return", "sbyte", "select", "short", "sizeof", "stackalloc", "string",
"switch", "throw", "true", "try", "typeof", "uint", "ulong", "ushort", "var", "where", "while", "yield",
"abstract", "async", "base", "class", "delegate", "enum", "explicit", "extern", "fixed", "get",
"implicit", "interface", "internal", "namespace", "operator", "override", "params", "private", "protected", "public",
"using", "partial", "readonly", "sealed", "set", "static", "struct", "this", "unchecked", "unsafe", "value", "virtual", "volatile", "void" };
public override Color HighlightColor => highlightColor;
public Color highlightColor = new Color(0.33f, 0.61f, 0.83f, 1.0f);
private readonly HashSet<string> shortlist = new HashSet<string>();
private readonly Stack<string> removeList = new Stack<string>();
public override bool IsImplicitMatch(CSLexer lexer)
{
if (!char.IsWhiteSpace(lexer.Previous) &&
!lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
{
return false;
}
shortlist.Clear();
int currentIndex = 0;
char currentChar = lexer.ReadNext();
for (int i = 0; i < Keywords.Length; i++)
{
if (Keywords[i][0] == currentChar)
shortlist.Add(Keywords[i]);
}
if (shortlist.Count == 0)
return false;
do
{
if (lexer.EndOfStream)
{
RemoveLongStrings(currentIndex + 1);
break;
}
currentChar = lexer.ReadNext();
currentIndex++;
if (char.IsWhiteSpace(currentChar) ||
lexer.IsSpecialSymbol(currentChar, DelimiterType.Start))
{
RemoveLongStrings(currentIndex);
lexer.Rollback(1);
break;
}
foreach (string keyword in shortlist)
{
if (currentIndex >= keyword.Length || keyword[currentIndex] != currentChar)
removeList.Push(keyword);
}
while (removeList.Count > 0)
shortlist.Remove(removeList.Pop());
}
while (shortlist.Count > 0);
return shortlist.Count > 0;
}
private void RemoveLongStrings(int length)
{
foreach (string keyword in shortlist)
{
if (keyword.Length > length)
removeList.Push(keyword);
}
while (removeList.Count > 0)
shortlist.Remove(removeList.Pop());
}
}
}

18
src/UI/CSConsole/Lexer/Lexer.cs Normal file
View File

@ -0,0 +1,18 @@
using System.Collections.Generic;
using UnityEngine;
using System.Linq;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public abstract class Lexer
{
public virtual IEnumerable<char> Delimiters => Enumerable.Empty<char>();
protected abstract Color HighlightColor { get; }
public string ColorTag => colorTag ?? (colorTag = "<color=#" + HighlightColor.ToHex() + ">");
private string colorTag;
public abstract bool TryMatchCurrent(LexerBuilder lexer);
}
}

31
src/UI/CSConsole/Lexer/Matcher.cs
View File

@ -1,31 +0,0 @@
using System.Collections.Generic;
using UnityEngine;
using System.Linq;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public abstract class Matcher
{
public abstract Color HighlightColor { get; }
public string HexColorTag => htmlColor ?? (htmlColor = "<color=#" + HighlightColor.ToHex() + ">");
private string htmlColor;
public virtual IEnumerable<char> StartChars => Enumerable.Empty<char>();
public virtual IEnumerable<char> EndChars => Enumerable.Empty<char>();
public abstract bool IsImplicitMatch(CSLexer lexer);
public bool IsMatch(CSLexer lexer)
{
if (IsImplicitMatch(lexer))
{
lexer.Commit();
return true;
}
lexer.Rollback();
return false;
}
}
}

31
src/UI/CSConsole/Lexer/NumberLexer.cs Normal file
View File

@ -0,0 +1,31 @@
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public class NumberLexer : Lexer
{
protected override Color HighlightColor => new Color(0.58f, 0.33f, 0.33f, 1.0f);
private bool IsNumeric(char c) => char.IsNumber(c) || c == '.';
public override bool TryMatchCurrent(LexerBuilder lexer)
{
// previous character must be whitespace or delimiter
if (!lexer.IsDelimiter(lexer.Previous, true))
return false;
if (!IsNumeric(lexer.Current))
return false;
while (!lexer.EndOfInput)
{
lexer.Commit();
if (!IsNumeric(lexer.PeekNext()))
break;
}
return true;
}
}
}

39
src/UI/CSConsole/Lexer/NumberMatch.cs
View File

@ -1,39 +0,0 @@
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public class NumberMatch : Matcher
{
public override Color HighlightColor => new Color(0.58f, 0.33f, 0.33f, 1.0f);
public override bool IsImplicitMatch(CSLexer lexer)
{
if (!char.IsWhiteSpace(lexer.Previous) &&
!lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
{
return false;
}
bool matchedNumber = false;
while (!lexer.EndOfStream)
{
if (IsNumberOrDecimalPoint(lexer.ReadNext()))
{
matchedNumber = true;
lexer.Commit();
}
else
{
lexer.Rollback();
break;
}
}
return matchedNumber;
}
private bool IsNumberOrDecimalPoint(char character) => char.IsNumber(character) || character == '.';
}
}

59
src/UI/CSConsole/Lexer/StringLexer.cs Normal file
View File

@ -0,0 +1,59 @@
using System.Collections.Generic;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public class StringLexer : Lexer
{
protected override Color HighlightColor => new Color(0.79f, 0.52f, 0.32f, 1.0f);
public override IEnumerable<char> Delimiters => new[] { '"' };
public override bool TryMatchCurrent(LexerBuilder lexer)
{
if (lexer.Current != '"')
return false;
if (lexer.Previous == '@')
{
// verbatim string, continue until un-escaped quote.
while (!lexer.EndOfInput)
{
lexer.Commit();
if (lexer.PeekNext() == '"')
{
lexer.Commit();
// possibly the end, check for escaped quotes.
// commit the character and flip the escape bool for each quote.
bool escaped = false;
while (lexer.PeekNext() == '"')
{
lexer.Commit();
escaped = !escaped;
}
// if the last quote wasnt escaped, that was the end of the string.
if (!escaped)
break;
}
}
}
else
{
// normal string
// continue until a quote which is not escaped, or end of input
while (!lexer.EndOfInput)
{
lexer.Commit();
if (lexer.PeekNext() == '"' && lexer.Previous != '\\')
{
lexer.Commit();
break;
}
}
}
return true;
}
}
}

26
src/UI/CSConsole/Lexer/StringMatch.cs
View File

@ -1,26 +0,0 @@
using System.Collections.Generic;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public class StringMatch : Matcher
{
public override Color HighlightColor => new Color(0.79f, 0.52f, 0.32f, 1.0f);
public override IEnumerable<char> StartChars => new[] { '"' };
public override IEnumerable<char> EndChars => new[] { '"' };
public override bool IsImplicitMatch(CSLexer lexer)
{
if (lexer.ReadNext() == '"')
{
while (!IsClosingQuoteOrEndFile(lexer, lexer.ReadNext())) { }
return true;
}
return false;
}
private bool IsClosingQuoteOrEndFile(CSLexer lexer, char character) => lexer.EndOfStream || character == '"';
}
}

52
src/UI/CSConsole/Lexer/SymbolLexer.cs Normal file
View File

@ -0,0 +1,52 @@
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexers
{
public class SymbolLexer : Lexer
{
protected override Color HighlightColor => Color.white;
// all symbols are delimiters
public override IEnumerable<char> Delimiters => uniqueSymbols;
// all symbol combinations are made of valid individual symbols.
private readonly HashSet<char> uniqueSymbols = new HashSet<char>
{
'[', ']', '{', '}', '(', ')', ',', '.', ';', ':',
'+', '-', '*', '/', '%', '&', '|', '^', '~', '=',
'<', '>', '?', '!', '@'
};
// // actual valid symbol combinations
// private readonly HashSet<string> actualSymbols = new HashSet<string>
// {
//"[", "]", "(", ")", "{", "}", ".", ",", ";", ":", "+", "-", "*", "/", "%", "&", "|", "^", "~", "=",
//"<", ">", "++", "--", "&&", "||", "<<", ">>", "==", "!=", "<=", ">=", "+=", "-=", "*=", "/=", "%=",
//"&=", "|=", "^=", "<<=", ">>=", "->", "!", "?", "??", "@", "=>",
// };
public override bool TryMatchCurrent(LexerBuilder lexer)
{
// previous character must be delimiter, whitespace, or alphanumeric.
if (!lexer.IsDelimiter(lexer.Previous, true, true))
return false;
if (uniqueSymbols.Contains(lexer.Current))
{
do
{
lexer.Commit();
lexer.PeekNext();
}
while (uniqueSymbols.Contains(lexer.Current));
return true;
}
return false;
}
}
}

98
src/UI/CSConsole/Lexer/SymbolMatch.cs
View File

@ -1,98 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
namespace UnityExplorer.UI.CSharpConsole.Lexer
{
public class SymbolMatch : Matcher
{
public override Color HighlightColor => new Color(0.58f, 0.47f, 0.37f, 1.0f);
private readonly string[] symbols = new[]
{
"[", "]", "(", ")", ".", "?", ":", "+", "-", "*", "/", "%", "&", "|", "^", "~", "=", "<", ">",
"++", "--", "&&", "||", "<<", ">>", "==", "!=", "<=", ">=", "+=", "-=", "*=", "/=", "%=", "&=",
"|=", "^=", "<<=", ">>=", "->", "??", "=>",
};
private static readonly List<string> shortlist = new List<string>();
private static readonly Stack<string> removeList = new Stack<string>();
public override IEnumerable<char> StartChars => symbols.Select(s => s[0]);
public override IEnumerable<char> EndChars => symbols.Select(s => s[0]);
public override bool IsImplicitMatch(CSLexer lexer)
{
if (lexer == null)
return false;
if (!char.IsWhiteSpace(lexer.Previous) &&
!char.IsLetter(lexer.Previous) &&
!char.IsDigit(lexer.Previous) &&
!lexer.IsSpecialSymbol(lexer.Previous, DelimiterType.End))
{
return false;
}
shortlist.Clear();
int currentIndex = 0;
char currentChar = lexer.ReadNext();
for (int i = symbols.Length - 1; i >= 0; i--)
{
if (symbols[i][0] == currentChar)
shortlist.Add(symbols[i]);
}
if (shortlist.Count == 0)
return false;
do
{
if (lexer.EndOfStream)
{
RemoveLongStrings(currentIndex + 1);
break;
}
currentChar = lexer.ReadNext();
currentIndex++;
if (char.IsWhiteSpace(currentChar) ||
char.IsLetter(currentChar) ||
char.IsDigit(currentChar) ||
lexer.IsSpecialSymbol(currentChar, DelimiterType.Start))
{
RemoveLongStrings(currentIndex);
lexer.Rollback(1);
break;
}
foreach (string symbol in shortlist)
{
if (currentIndex >= symbol.Length || symbol[currentIndex] != currentChar)
removeList.Push(symbol);
}
while (removeList.Count > 0)
shortlist.Remove(removeList.Pop());
}
while (shortlist.Count > 0);
return shortlist.Count > 0;
}
private void RemoveLongStrings(int length)
{
foreach (string keyword in shortlist)
{
if (keyword.Length > length)
removeList.Push(keyword);
}
while (removeList.Count > 0)
shortlist.Remove(removeList.Pop());
}
}
}

347
src/UI/CSConsole/LexerBuilder.cs Normal file
View File

@ -0,0 +1,347 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using UnityEngine;
using UnityEngine.UI;
using UnityExplorer.UI.CSharpConsole.Lexers;
namespace UnityExplorer.UI.CSharpConsole
{
public struct MatchInfo
{
public int startIndex;
public int endIndex;
public string htmlColorTag;
}
public class LexerBuilder
{
// Initialization and core
public const char WHITESPACE = ' ';
public const char INDENT_OPEN = '{';
public const char INDENT_CLOSE = '}';
private readonly Lexer[] lexers;
private readonly HashSet<char> delimiters = new HashSet<char>();
public LexerBuilder()
{
lexers = new Lexer[]
{
new CommentLexer(),
new SymbolLexer(),
new StringLexer(),
new NumberLexer(),
new KeywordLexer(),
};
foreach (var matcher in lexers)
{
foreach (char c in matcher.Delimiters)
{
if (!delimiters.Contains(c))
delimiters.Add(c);
}
}
}
public bool IsDelimiter(char character, bool orWhitespace = false, bool orLetterOrDigit = false)
{
return delimiters.Contains(character)
|| (orWhitespace && char.IsWhiteSpace(character))
|| (orLetterOrDigit && char.IsLetterOrDigit(character));
}
// Lexer enumeration
public string InputString { get; private set; }
public int Length => InputString.Length;
public int LastCommittedIndex { get; private set; }
public int LookaheadIndex { get; private set; }
public char Current => !EndOfInput ? InputString[LookaheadIndex] : WHITESPACE;
public char Previous => LookaheadIndex >= 1 ? InputString[LookaheadIndex - 1] : WHITESPACE;
public bool EndOfInput => LookaheadIndex >= Length;
public bool EndOrNewLine => EndOfInput || Current == '\n' || Current == '\r';
public string SyntaxHighlight(string input)
{
var sb = new StringBuilder();
int lastUnhighlighted = 0;
// TODO auto indent as part of this parse
foreach (var match in GetMatches(input))
{
// append non-highlighted text between last match and this
for (int i = lastUnhighlighted; i < match.startIndex; i++)
sb.Append(input[i]);
// append the highlighted match
sb.Append(match.htmlColorTag);
for (int i = match.startIndex; i <= match.endIndex && i < input.Length; i++)
sb.Append(input[i]);
sb.Append(SignatureHighlighter.CLOSE_COLOR);
// update the last unhighlighted start index
lastUnhighlighted = match.endIndex + 1;
}
return sb.ToString();
}
public IEnumerable<MatchInfo> GetMatches(string input)
{
if (string.IsNullOrEmpty(input))
yield break;
InputString = input;
LastCommittedIndex = -1;
Rollback();
while (!EndOfInput)
{
SkipWhitespace();
bool anyMatch = false;
int startIndex = LastCommittedIndex + 1;
foreach (var lexer in lexers)
{
if (lexer.TryMatchCurrent(this))
{
anyMatch = true;
yield return new MatchInfo
{
startIndex = startIndex,
endIndex = LastCommittedIndex,
htmlColorTag = lexer.ColorTag,
};
break;
}
else
Rollback();
}
if (!anyMatch)
{
LookaheadIndex = LastCommittedIndex + 1;
Commit();
}
}
}
public char PeekNext(int amount = 1)
{
LookaheadIndex += amount;
return Current;
}
public void Commit()
{
LastCommittedIndex = Math.Min(Length - 1, LookaheadIndex);
}
public void Rollback()
{
LookaheadIndex = LastCommittedIndex + 1;
}
public void RollbackBy(int amount)
{
LookaheadIndex = Math.Max(LastCommittedIndex + 1, LookaheadIndex - amount);
}
private void SkipWhitespace()
{
// peek and commit as long as there is whitespace
while (!EndOfInput && char.IsWhiteSpace(Current))
{
Commit();
PeekNext();
}
// revert the last PeekNext which would have returned false
Rollback();
}
#region AUTO INDENT TODO
// Auto-indenting
//public int GetIndentLevel(string input, int toIndex)
//{
// bool stringState = false;
// int indent = 0;
//
// for (int i = 0; i < toIndex && i < input.Length; i++)
// {
// char character = input[i];
//
// if (character == '"')
// stringState = !stringState;
// else if (!stringState && character == INDENT_OPEN)
// indent++;
// else if (!stringState && character == INDENT_CLOSE)
// indent--;
// }
//
// if (indent < 0)
// indent = 0;
//
// return indent;
//}
//// TODO not quite correct, but almost there.
//
//public string AutoIndentOnEnter(string input, ref int caretPos)
//{
// var sb = new StringBuilder(input);
//
// bool inString = false;
// bool inChar = false;
// int currentIndent = 0;
// int curLineIndent = 0;
// bool prevWasNewLine = true;
//
// // process before caret position
// for (int i = 0; i < caretPos; i++)
// {
// char c = sb[i];
//
// ExplorerCore.Log(i + ": " + c);
//
// // update string/char state
// if (!inChar && c == '\"')
// inString = !inString;
// else if (!inString && c == '\'')
// inChar = !inChar;
//
// // continue if inside string or char
// if (inString || inChar)
// continue;
//
// // check for new line
// if (c == '\n')
// {
// ExplorerCore.Log("new line, resetting line counts");
// curLineIndent = 0;
// prevWasNewLine = true;
// }
// // check for indent
// else if (c == '\t' && prevWasNewLine)
// {
// ExplorerCore.Log("its a tab");
// if (curLineIndent > currentIndent)
// {
// ExplorerCore.Log("too many tabs, removing");
// // already reached the indent we should have
// sb.Remove(i, 1);
// i--;
// caretPos--;
// curLineIndent--;
// }
// else
// curLineIndent++;
// }
// // remove spaces on new lines
// else if (c == ' ' && prevWasNewLine)
// {
// ExplorerCore.Log("removing newline-space");
// sb.Remove(i, 1);
// i--;
// caretPos--;
// }
// else
// {
// if (c == INDENT_CLOSE)
// currentIndent--;
//
// if (prevWasNewLine && curLineIndent < currentIndent)
// {
// ExplorerCore.Log("line is not indented enough");
// // line is not indented enough
// int diff = currentIndent - curLineIndent;
// sb.Insert(i, new string('\t', diff));
// caretPos += diff;
// i += diff;
// }
//
// // check for brackets
// if ((c == INDENT_CLOSE || c == INDENT_OPEN) && !prevWasNewLine)
// {
// ExplorerCore.Log("bracket needs new line");
//
// // need to put it on a new line
// sb.Insert(i, $"\n{new string('\t', currentIndent)}");
// caretPos += 1 + currentIndent;
// i += 1 + currentIndent;
// }
//
// if (c == INDENT_OPEN)
// currentIndent++;
//
// prevWasNewLine = false;
// }
// }
//
// // todo put caret on new line after previous bracket if needed
// // indent caret to current indent
//
// // process after caret position, make sure there are equal opened/closed brackets
// ExplorerCore.Log("-- after caret --");
// for (int i = caretPos; i < sb.Length; i++)
// {
// char c = sb[i];
// ExplorerCore.Log(i + ": " + c);
//
// // update string/char state
// if (!inChar && c == '\"')
// inString = !inString;
// else if (!inString && c == '\'')
// inChar = !inChar;
//
// if (inString || inChar)
// continue;
//
// if (c == INDENT_OPEN)
// currentIndent++;
// else if (c == INDENT_CLOSE)
// currentIndent--;
// }
//
// if (currentIndent > 0)
// {
// ExplorerCore.Log("there are not enough closing brackets, curIndent is " + currentIndent);
// // There are not enough close brackets
//
// // TODO this should append in reverse indent order (small indents inserted first, then biggest).
// while (currentIndent > 0)
// {
// ExplorerCore.Log("Inserting closing bracket with " + currentIndent + " indent");
// // append the indented '}' on a new line
// sb.Insert(caretPos, $"\n{new string('\t', currentIndent - 1)}}}");
//
// currentIndent--;
// }
//
// }
// //else if (currentIndent < 0)
// //{
// // // There are too many close brackets
// //
// // // todo?
// //}
//
// return sb.ToString();
//}
#endregion
}
}

3
src/UI/Panels/CSConsolePanel.cs
View File

@ -159,7 +159,7 @@ namespace UnityExplorer.UI.Panels
InputText = InputField.InputField.textComponent;
InputText.supportRichText = false;
InputText.color = new Color(1, 1, 1, 0.5f);
InputText.color = new Color(1, 1, 1, 0.65f);
var mainTextObj = InputText.gameObject;
var highlightTextObj = UIFactory.CreateUIObject("HighlightText", mainTextObj.gameObject);
@ -171,6 +171,7 @@ namespace UnityExplorer.UI.Panels
highlightTextRect.offsetMax = new Vector2(14, 0);
HighlightText = highlightTextObj.AddComponent<Text>();
HighlightText.color = Color.clear;
HighlightText.supportRichText = true;
HighlightText.fontSize = fontSize;

2
src/UI/UIManager.cs
View File

@ -119,9 +119,7 @@ namespace UnityExplorer.UI
UIPanel.UpdateFocus();
PanelDragger.UpdateInstances();
UIBehaviourModel.UpdateInstances();
AutoCompleter.Update();
}
public static void TogglePanel(Panels panel)

14
src/UnityExplorer.csproj
View File

@ -219,13 +219,13 @@
</ItemGroup>
<ItemGroup>
<Compile Include="Core\Config\InternalConfigHandler.cs" />
<Compile Include="UI\CSConsole\CSLexer.cs" />
<Compile Include="UI\CSConsole\Lexer\CommentMatch.cs" />
<Compile Include="UI\CSConsole\Lexer\KeywordMatch.cs" />
<Compile Include="UI\CSConsole\Lexer\Matcher.cs" />
<Compile Include="UI\CSConsole\Lexer\NumberMatch.cs" />
<Compile Include="UI\CSConsole\Lexer\StringMatch.cs" />
<Compile Include="UI\CSConsole\Lexer\SymbolMatch.cs" />
<Compile Include="UI\CSConsole\LexerBuilder.cs" />
<Compile Include="UI\CSConsole\Lexer\CommentLexer.cs" />
<Compile Include="UI\CSConsole\Lexer\KeywordLexer.cs" />
<Compile Include="UI\CSConsole\Lexer\Lexer.cs" />
<Compile Include="UI\CSConsole\Lexer\NumberLexer.cs" />
<Compile Include="UI\CSConsole\Lexer\StringLexer.cs" />
<Compile Include="UI\CSConsole\Lexer\SymbolLexer.cs" />
<Compile Include="UI\CSConsole\ScriptEvaluator.cs" />
<Compile Include="UI\CSConsole\ScriptInteraction.cs" />
<Compile Include="Core\ExplorerBehaviour.cs" />