Added all tokens to unit test

lexer/lexer_test.go

@@ -4,20 +4,52 @@ import "testing"
 import "github.com/sashakoshka/arf/file"
 
 func TestTokenizeAll (test *testing.T) {
-	file, err := file.Open("tests/parser/all")
+	file, err := file.Open("tests/lexer/all")
 	if err != nil {
 		test.Log(err)
 		test.Fail()
 	}
 	
 	tokens, err := Tokenize(file)
-	if err != nil {
+	if err == nil {
-		test.Log(err)
+		test.Log("Tokenize() have returned an error")
 		test.Fail()
 	}
 
 	correct := []Token {
-		Token { kind: TokenKindSeparator, },
+		Token { kind: TokenKindSeparator },
+		Token { kind: TokenKindPermission /* TODO: value */ },
+		Token { kind: TokenKindReturnDirection },
+		Token { kind: TokenKindInt, value: -349820394 },
+		Token { kind: TokenKindUInt, value: 932748397 },
+		Token { kind: TokenKindFloat, value: 239485.37520 },
+		Token { kind: TokenKindString, value: "hello world\n" },
+		Token { kind: TokenKindRune, value: 'E' },
+		Token { kind: TokenKindName, value: "helloWorld" },
+		Token { kind: TokenKindColon },
+		Token { kind: TokenKindDot },
+		Token { kind: TokenKindLBracket },
+		Token { kind: TokenKindRBracket },
+		Token { kind: TokenKindLBrace },
+		Token { kind: TokenKindRBrace },
+		Token { kind: TokenKindPlus },
+		Token { kind: TokenKindMinus },
+		Token { kind: TokenKindIncrement },
+		Token { kind: TokenKindDecrement },
+		Token { kind: TokenKindAsterisk },
+		Token { kind: TokenKindSlash },
+		Token { kind: TokenKindAt },
+		Token { kind: TokenKindExclamation },
+		Token { kind: TokenKindPercent },
+		Token { kind: TokenKindTilde },
+		Token { kind: TokenKindLessThan },
+		Token { kind: TokenKindLShift },
+		Token { kind: TokenKindGreaterThan },
+		Token { kind: TokenKindRShift },
+		Token { kind: TokenKindBinaryOr },
+		Token { kind: TokenKindLogicalOr },
+		Token { kind: TokenKindBinaryAnd },
+		Token { kind: TokenKindLogicalAnd },
 	}
 
 	if len(tokens) != len(correct) {

lexer/token.go

@@ -14,12 +14,12 @@ const (
         TokenKindReturnDirection
 
         TokenKindInt
+        TokenKindUInt
         TokenKindFloat
         TokenKindString
         TokenKindRune
 
         TokenKindName
-        TokenKindSymbol
 
         TokenKindColon
         TokenKindDot
@@ -31,6 +31,8 @@ const (
 
         TokenKindPlus
         TokenKindMinus
+        TokenKindIncrement
+        TokenKindDecrement
         TokenKindAsterisk
         TokenKindSlash
 

tests/lexer/all

@@ -0,0 +1,2 @@
+--- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.[]{}
++ - ++ -- * / @ ! % ~ < << > >> | || & &&

tests/parser/all

@@ -1,6 +0,0 @@
-# normal tokens
---- rn rr nw -> 239 -120 + - / * -- ++ [{[skadjsk 2039.32]}] 0x5FC2D
-"skdjlsakdj" '\n'
-
-# erroneous tokens (poorly spaced)
----rnrrnw->239-120+-/*--++