Added support for hex, octal, and unicode escape sequences

lexer/lexer_test.go

@@ -133,5 +133,7 @@ func TestTokenizeText (test *testing.T) {
 		Token { kind: TokenKindRune, value: '"'  },
 		Token { kind: TokenKindRune, value: '\\' },
 		Token { kind: TokenKindNewline },
+		Token { kind: TokenKindString, value: "hello world \x40\u0040\U00000040!" },
+		Token { kind: TokenKindNewline },
 	}, test)
 }

lexer/text.go

@@ -1,20 +1,9 @@
 package lexer
 
+import "strconv"
 import "github.com/sashakoshka/arf/file"
 
-var escapeSequenceMap = map[rune] rune {
+// tokenizeString tokenizes a string or rune literal.
-        'a':  '\x07',
-        'b':  '\x08',
-        'f':  '\x0c',
-        'n':  '\x0a',
-        'r':  '\x0d',
-        't':  '\x09',
-        'v':  '\x0b',
-        '\'': '\'',
-        '"':  '"',
-        '\\': '\\',
-}
-
 func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) {
 	err = lexer.nextRune()
 	if err != nil { return }