From d0c844972dabdc7b1e43a698cb2bdae7fa4165c0 Mon Sep 17 00:00:00 2001 From: Clemens-Dautermann Date: Mon, 17 Aug 2020 23:26:53 +0200 Subject: [PATCH] Error Fixes - Lexer discarding all newlines (now replaces it with spaces) - Cleanup of Type check and remove function in parser - Cleanup in Compiler.cs --- .../.idea.Compiler/.idea/contentModel.xml | 3 +- .../.idea/.idea.Compiler/.idea/workspace.xml | 136 ++++++++++-------- Compiler/Compiler.cs | 62 +++++--- Compiler/Lexer/Lexer.cs | 4 +- Compiler/Lexer/TokenType.cs | 8 +- .../Exceptions/MissingTokenException.cs | 15 ++ .../Exceptions/UnexpectedTokenException.cs | 7 +- Compiler/Parser/Parser.cs | 109 +++++++------- 8 files changed, 198 insertions(+), 146 deletions(-) create mode 100644 Compiler/Parser/Exceptions/MissingTokenException.cs diff --git a/Compiler/.idea/.idea.Compiler/.idea/contentModel.xml b/Compiler/.idea/.idea.Compiler/.idea/contentModel.xml index dc3a832..6dcee0f 100644 --- a/Compiler/.idea/.idea.Compiler/.idea/contentModel.xml +++ b/Compiler/.idea/.idea.Compiler/.idea/contentModel.xml @@ -2,7 +2,7 @@ - + @@ -17,6 +17,7 @@ + diff --git a/Compiler/.idea/.idea.Compiler/.idea/workspace.xml b/Compiler/.idea/.idea.Compiler/.idea/workspace.xml index bb38169..6a42945 100644 --- a/Compiler/.idea/.idea.Compiler/.idea/workspace.xml +++ b/Compiler/.idea/.idea.Compiler/.idea/workspace.xml @@ -20,8 +20,13 @@ + + + + + + + + + + + + + + + + + - - - - - - - - - - - - @@ -101,8 +107,8 @@ - - + + @@ -146,6 +152,8 @@ + + @@ -179,94 +187,94 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + @@ -275,10 +283,10 @@ - + - + @@ -291,5 +299,9 @@ + + + + \ No newline at end of file diff --git a/Compiler/Compiler.cs b/Compiler/Compiler.cs index 2b63cc3..ae27887 100644 --- a/Compiler/Compiler.cs +++ b/Compiler/Compiler.cs @@ -1,37 +1,49 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Compiler.Lexer; using Compiler.Parser; +using Compiler.Parser.Nodes; namespace Compiler { - public class Compiler + public static class Compiler { public static void Main() { - List> tokensValid = TestLexer("../../../../tests/week_1/valid"); - List> tokensInvalid = TestLexer("../../../../tests/week_1/invalid"); + string[] validFiles = Directory.GetFiles("../../../../tests/week_1/valid"); + string[] invalidFiles = Directory.GetFiles("../../../../tests/week_1/invalid"); - TestParser(tokensValid[0]); - TestParser(tokensInvalid[0]); + foreach (string filepath in validFiles) + { + List tokens = TestLexer(filepath, 0); + TestParser(tokens, filepath); + } + + foreach (string filepath in invalidFiles) + { + List tokens = TestLexer(filepath, 0); + TestParser(tokens, filepath); + } } - static List> TestLexer(String path) + static List TestLexer(string path, int debug) { - List> tokenLists = new List>(); - String[] files = Directory.GetFiles(path); + //List> tokenLists = new List>(); + //string[] files = Directory.GetFiles(path); Lexer.Lexer lexer = new Lexer.Lexer(); - foreach (String filename in files) + + StreamReader file = new StreamReader(path); + string contents = file.ReadToEnd(); + + List tokens = lexer.Lex(contents); + Console.WriteLine("Lexed \"" + path.Split("/").Last() + "\""); + + if (debug > 0) { - StreamReader file = new StreamReader(filename); - String contents = file.ReadToEnd(); - - List tokens = lexer.Lex(contents); - tokenLists.Add(tokens); - - Console.WriteLine("-----------" + filename + "-----------"); + Console.WriteLine("-----------" + path + "-----------"); foreach (Token token in tokens) { Console.WriteLine(token.ToString()); @@ -40,14 +52,24 @@ namespace Compiler Console.WriteLine("--------------------------------------"); } - return tokenLists; + + return tokens; } - static void TestParser(List tokenList) + static void TestParser(List tokenList, string path) { - Parser.Parser p = new Parser.Parser(); + Parser.Parser p = new Parser.Parser(tokenList); - p.Parse(ref tokenList, NodeType.ProgramNode); + try + { + Node programNode = p.Parse(NodeType.ProgramNode); + Console.WriteLine("Parsed \"" + path.Split("/").Last() + "\""); + } + catch (Exception e) + { + Console.WriteLine("Error in file \"" + path.Split("/").Last() + "\""); + Console.WriteLine(e.Message); + } } } } \ No newline at end of file diff --git a/Compiler/Lexer/Lexer.cs b/Compiler/Lexer/Lexer.cs index 0107256..4d2e4d3 100644 --- a/Compiler/Lexer/Lexer.cs +++ b/Compiler/Lexer/Lexer.cs @@ -10,8 +10,8 @@ namespace Compiler.Lexer public List Lex(String inputString) { //strip newlines - inputString = inputString.Replace("\n", ""); - inputString = inputString.Replace("\r", ""); + inputString = inputString.Replace("\n", " "); + inputString = inputString.Replace("\r", " "); //initialize Token List List tokens = new List(); diff --git a/Compiler/Lexer/TokenType.cs b/Compiler/Lexer/TokenType.cs index df302f7..24a558c 100644 --- a/Compiler/Lexer/TokenType.cs +++ b/Compiler/Lexer/TokenType.cs @@ -11,9 +11,11 @@ namespace Compiler.Lexer SemicolonToken, IdentifierToken, IntegerLiteralToken, - + //special Token to represent invalid matches - InvalidToken - + InvalidToken, + + //empty toke + NullToken } } \ No newline at end of file diff --git a/Compiler/Parser/Exceptions/MissingTokenException.cs b/Compiler/Parser/Exceptions/MissingTokenException.cs new file mode 100644 index 0000000..709a7f8 --- /dev/null +++ b/Compiler/Parser/Exceptions/MissingTokenException.cs @@ -0,0 +1,15 @@ +using System; +using Compiler.Lexer; + +namespace Compiler.Parser.Exceptions +{ + public class MissingTokenException : Exception + { + public override string Message { get; } + + public MissingTokenException(TokenType expected) + { + this.Message = "Expected Token " + expected + " missing."; + } + } +} \ No newline at end of file diff --git a/Compiler/Parser/Exceptions/UnexpectedTokenException.cs b/Compiler/Parser/Exceptions/UnexpectedTokenException.cs index aa51c77..a8ce7ce 100644 --- a/Compiler/Parser/Exceptions/UnexpectedTokenException.cs +++ b/Compiler/Parser/Exceptions/UnexpectedTokenException.cs @@ -5,16 +5,11 @@ namespace Compiler.Parser.Exceptions { public class UnexpectedTokenException : Exception { - public TokenType expected { get; set; } - public TokenType got { get; set; } - public override string Message { get; } public UnexpectedTokenException(TokenType expected, TokenType got) { - this.expected = expected; - this.got = got; - this.Message = "Unexpected Token " + got + ". Expected: " + expected; + this.Message = "Unexpected Token " + got + ", expected: " + expected + "."; } } } \ No newline at end of file diff --git a/Compiler/Parser/Parser.cs b/Compiler/Parser/Parser.cs index 9f912be..cff73fe 100644 --- a/Compiler/Parser/Parser.cs +++ b/Compiler/Parser/Parser.cs @@ -9,11 +9,30 @@ namespace Compiler.Parser { public class Parser { - public Node Parse(ref List tokenList, NodeType nodeType) - { - var t = tokenList.FirstOrDefault(); + private readonly List _tokenList; - if (t == null) throw new Exception("Empty Token List."); + public Parser(List tokenList) + { + _tokenList = tokenList; + } + + private void CheckFirstTokenAndRemove(TokenType expected) + { + if (_tokenList.Count == 0) + { + throw new MissingTokenException(expected); + } + + if (_tokenList[0].TokenType != expected) + { + throw new UnexpectedTokenException(expected, _tokenList[0].TokenType); + } + + _tokenList.RemoveAt(0); + } + + public Node Parse(NodeType nodeType) + { Node n; switch (nodeType) { @@ -21,32 +40,31 @@ namespace Compiler.Parser n = new ProgramNode(); //if this node is a program node, the next one must be a function node - Node childNode = Parse(ref tokenList, NodeType.FunctionNode); + Node childNode = Parse(NodeType.FunctionNode); n.Children.Add(childNode); break; case NodeType.FunctionNode: n = new FunctionNode(); - //retrieve signature and remove it - List signature = tokenList.GetRange(0, 5); - tokenList.RemoveRange(0, 5); - //check each element of the signature and raise corresponding errors - if (signature[0].TokenType != TokenType.IntToken) + + CheckFirstTokenAndRemove(TokenType.IntToken); + + if (_tokenList.Count == 0) { - throw new UnexpectedTokenException(TokenType.IntToken, signature[1].TokenType); + throw new MissingTokenException(TokenType.IdentifierToken); } - if (signature[1].Value != null) + if (_tokenList[0].Value != null) { - if (signature[1].TokenType == TokenType.IdentifierToken) + if (_tokenList[0].TokenType == TokenType.IdentifierToken) { - ((FunctionNode) n).Name = signature[1].Value.ToString(); + ((FunctionNode) n).Name = _tokenList[0].Value.ToString(); } else { - throw new UnexpectedTokenException(TokenType.IdentifierToken, signature[1].TokenType); + throw new UnexpectedTokenException(TokenType.IdentifierToken, _tokenList[0].TokenType); } } else @@ -54,64 +72,48 @@ namespace Compiler.Parser throw new InvalidIdentifierException(null); } - if (signature[2].TokenType != TokenType.OpenParenthesisToken) - { - throw new UnexpectedTokenException(TokenType.OpenParenthesisToken, signature[1].TokenType); - } + //remove + _tokenList.RemoveAt(0); - if (signature[3].TokenType != TokenType.CloseParenthesisToken) - { - throw new UnexpectedTokenException(TokenType.CloseParenthesisToken, signature[1].TokenType); - } - - if (signature[4].TokenType != TokenType.OpenBraceToken) - { - throw new UnexpectedTokenException(TokenType.OpenBraceToken, signature[1].TokenType); - } + CheckFirstTokenAndRemove(TokenType.OpenParenthesisToken); + CheckFirstTokenAndRemove(TokenType.CloseParenthesisToken); + CheckFirstTokenAndRemove(TokenType.OpenBraceToken); //add returned child node to AST - n.Children.Add(Parse(ref tokenList, NodeType.StatementNode)); + n.Children.Add(Parse(NodeType.StatementNode)); //remove trailing } - if (tokenList[0].TokenType != TokenType.CloseBraceToken) - { - throw new UnexpectedTokenException(TokenType.CloseBraceToken, tokenList[0].TokenType); - } + CheckFirstTokenAndRemove(TokenType.CloseBraceToken); - tokenList.RemoveAt(0); break; + case NodeType.StatementNode: //TODO: This Type of return/statement node will probably need fixing later n = new ReturnNode(); //get return token and remove it - List returnStatement = tokenList.GetRange(0, 1); - tokenList.RemoveAt(0); + CheckFirstTokenAndRemove(TokenType.ReturnToken); - if (returnStatement[0].TokenType != TokenType.ReturnToken) - { - throw new UnexpectedTokenException(TokenType.ReturnToken, returnStatement[0].TokenType); - } - else - { - //add returned child node to AST - n.Children.Add(Parse(ref tokenList, NodeType.ExpressionNode)); - //remove trailing ; + //add returned child node to AST + n.Children.Add(Parse(NodeType.ExpressionNode)); - if (tokenList[0].TokenType != TokenType.SemicolonToken) - { - throw new UnexpectedTokenException(TokenType.SemicolonToken, tokenList[0].TokenType); - } - tokenList.RemoveAt(0); - } + //remove trailing ; + CheckFirstTokenAndRemove(TokenType.SemicolonToken); + break; case NodeType.ExpressionNode: - Token constantToken = tokenList[0]; + if (_tokenList.Count == 0) + { + throw new MissingTokenException(TokenType.IntegerLiteralToken); + } + + Token constantToken = _tokenList[0]; + //check if TokenType is right if (constantToken.TokenType != TokenType.IntegerLiteralToken) { @@ -119,6 +121,9 @@ namespace Compiler.Parser } else { + //remove int literal token + _tokenList.RemoveAt(0); + //check if value Type is right if (constantToken.Value.GetType() != typeof(int)) {