From af6f1708334cd335228bdd6a94017aed482f6eec Mon Sep 17 00:00:00 2001
From: Sasha Koshka <sashakoshka@tebibyte.media>
Date: Thu, 11 Aug 2022 03:47:42 -0500
Subject: [PATCH] Normalized whitespace tokens

---
 lexer/lexer.go      | 19 +++++++++++++++++++
 lexer/lexer_test.go |  2 ++
 2 files changed, 21 insertions(+)

diff --git a/lexer/lexer.go b/lexer/lexer.go
index 1051d1b..742e981 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -54,6 +54,10 @@ func (lexer *LexingOperation) tokenize () (err error) {
 		if err != nil { return }
 	}
 
+	if lexer.tokens[len(lexer.tokens) - 1].kind != TokenKindNewline {
+		lexer.addToken(Token { kind: TokenKindNewline })
+	}
+
 	return
 }
 
@@ -119,6 +123,21 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 		}
 	case '\n':
 		// line break
+		lastLineEmpty := true
+		tokenIndex := len(lexer.tokens) - 1
+		for lexer.tokens[tokenIndex].kind != TokenKindNewline  {
+			if lexer.tokens[tokenIndex].kind != TokenKindIndent {
+
+				lastLineEmpty = false
+				break
+			}	
+			tokenIndex --
+		}
+
+		if lastLineEmpty {
+			lexer.tokens = lexer.tokens[:tokenIndex]
+		}
+		
 		// TODO: if last line was blank, (ony whitespace) discard.
 		lexer.addToken (Token {
 			kind: TokenKindNewline,
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
index 78d4f91..9865564 100644
--- a/lexer/lexer_test.go
+++ b/lexer/lexer_test.go
@@ -45,6 +45,7 @@ func TestTokenizeAll (test *testing.T) {
 		Token { kind: TokenKindRBracket },
 		Token { kind: TokenKindLBrace },
 		Token { kind: TokenKindRBrace },
+		Token { kind: TokenKindNewline },
 		Token { kind: TokenKindPlus },
 		Token { kind: TokenKindMinus },
 		Token { kind: TokenKindIncrement },
@@ -63,6 +64,7 @@ func TestTokenizeAll (test *testing.T) {
 		Token { kind: TokenKindLogicalOr },
 		Token { kind: TokenKindBinaryAnd },
 		Token { kind: TokenKindLogicalAnd },
+		Token { kind: TokenKindNewline },
 	}
 
 	if len(tokens) != len(correct) {