Normalized whitespace tokens

2022-08-11 03:47:42 -05:00
parent 0a31ea7bf8
commit af6f170833
2 changed files with 21 additions and 0 deletions
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -45,6 +45,7 @@ func TestTokenizeAll (test *testing.T) {
 		Token { kind: TokenKindRBracket },
 		Token { kind: TokenKindLBrace },
 		Token { kind: TokenKindRBrace },
+		Token { kind: TokenKindNewline },
 		Token { kind: TokenKindPlus },
 		Token { kind: TokenKindMinus },
 		Token { kind: TokenKindIncrement },
@@ -63,6 +64,7 @@ func TestTokenizeAll (test *testing.T) {
 		Token { kind: TokenKindLogicalOr },
 		Token { kind: TokenKindBinaryAnd },
 		Token { kind: TokenKindLogicalAnd },
+		Token { kind: TokenKindNewline },
 	}

 	if len(tokens) != len(correct) {