diff --git a/lexer/lexer.go b/lexer/lexer.go index c49154c..ea2c7d1 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -167,10 +167,8 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { token.kind = TokenKindNewline lexer.addToken(token) err = lexer.nextRune() - case '"': - err = lexer.tokenizeString(false) case '\'': - err = lexer.tokenizeString(true) + err = lexer.tokenizeString() case ':': token := lexer.newToken() token.kind = TokenKindColon diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 623410c..95dda44 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -82,7 +82,7 @@ func compareErr ( } _, err = Tokenize(file) - check := err.(infoerr.Error) + check, _ := err.(infoerr.Error) test.Log("error that was recieved:") test.Log(check) @@ -132,7 +132,7 @@ func TestTokenizeAll (test *testing.T) { quickToken(9, TokenKindUInt, uint64(932748397)), quickToken(12, TokenKindFloat, 239485.37520), quickToken(16, TokenKindString, "hello world!\n"), - quickToken(3, TokenKindRune, 'E'), + quickToken(3, TokenKindString, "E"), quickToken(10, TokenKindName, "helloWorld"), quickToken(1, TokenKindColon, nil), quickToken(1, TokenKindDot, nil), @@ -215,18 +215,17 @@ func TestTokenizeNumbers (test *testing.T) { func TestTokenizeText (test *testing.T) { checkTokenSlice("../tests/lexer/text.arf", test, - quickToken(34, TokenKindString, "hello world!\a\b\f\n\r\t\v'\"\\"), + quickToken(34, TokenKindString, "hello world!\a\b\f\n\r\t\v'\\"), quickToken(1, TokenKindNewline, nil), - quickToken(4, TokenKindRune, '\a'), - quickToken(4, TokenKindRune, '\b'), - quickToken(4, TokenKindRune, '\f'), - quickToken(4, TokenKindRune, '\n'), - quickToken(4, TokenKindRune, '\r'), - quickToken(4, TokenKindRune, '\t'), - quickToken(4, TokenKindRune, '\v'), - quickToken(4, TokenKindRune, '\''), - quickToken(4, TokenKindRune, '"' ), - quickToken(4, TokenKindRune, '\\'), + quickToken(4, TokenKindString, "\a"), + quickToken(4, TokenKindString, "\b"), + quickToken(4, TokenKindString, "\f"), + quickToken(4, TokenKindString, "\n"), + quickToken(4, TokenKindString, "\r"), + quickToken(4, TokenKindString, "\t"), + quickToken(4, TokenKindString, "\v"), + quickToken(4, TokenKindString, "'"), + quickToken(4, TokenKindString, "\\"), quickToken(1, TokenKindNewline, nil), quickToken(35, TokenKindString, "hello world \x40\u0040\U00000040!"), quickToken(1, TokenKindNewline, nil), diff --git a/lexer/text.go b/lexer/text.go index 8c0234e..8d68d50 100644 --- a/lexer/text.go +++ b/lexer/text.go @@ -4,7 +4,7 @@ import "strconv" import "git.tebibyte.media/arf/arf/infoerr" // tokenizeString tokenizes a string or rune literal. -func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) { +func (lexer *LexingOperation) tokenizeString () (err error) { err = lexer.nextRune() if err != nil { return } @@ -34,32 +34,15 @@ func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) { if err != nil { return } } - if isRuneLiteral { - if lexer.char == '\'' { break } - } else { - if lexer.char == '"' { break } - } + if lexer.char == '\'' { break } } err = lexer.nextRune() if err != nil { return } beginning.SetWidth(len(got)) - if isRuneLiteral { - if len(got) > 1 { - err = infoerr.NewError ( - beginning, - "excess data in rune literal", - infoerr.ErrorKindError) - return - } - - token.kind = TokenKindRune - token.value = rune([]rune(got)[0]) - } else { - token.kind = TokenKindString - token.value = got - } + token.kind = TokenKindString + token.value = got token.location.SetWidth(tokenWidth) lexer.addToken(token) @@ -77,7 +60,6 @@ var escapeSequenceMap = map[rune] rune { 't': '\x09', 'v': '\x0b', '\'': '\'', - '"': '"', '\\': '\\', } diff --git a/lexer/token.go b/lexer/token.go index 033bd06..dd52589 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -19,7 +19,6 @@ const ( TokenKindUInt TokenKindFloat TokenKindString - TokenKindRune TokenKindName @@ -156,8 +155,6 @@ func (tokenKind TokenKind) Describe () (description string) { description = "Float" case TokenKindString: description = "String" - case TokenKindRune: - description = "Rune" case TokenKindName: description = "Name" case TokenKindColon: diff --git a/tests/lexer/all.arf b/tests/lexer/all.arf index 950ece0..4f4d646 100644 --- a/tests/lexer/all.arf +++ b/tests/lexer/all.arf @@ -1,3 +1,3 @@ :arf ---- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,..()[]{} +--- rw -> -349820394 932748397 239485.37520 'hello world!\n' 'E' helloWorld:.,..()[]{} + - ++ -- * / @ ! % %= ~ ~= = == != < <= << <<= > >= >> >>= | |= || & &= && ^ ^= diff --git a/tests/lexer/error/unknownEscape.arf b/tests/lexer/error/unknownEscape.arf index cff9a06..5add659 100644 --- a/tests/lexer/error/unknownEscape.arf +++ b/tests/lexer/error/unknownEscape.arf @@ -1,2 +1,2 @@ :arf -"\g" +'\g' diff --git a/tests/lexer/text.arf b/tests/lexer/text.arf index abbc5e7..491401d 100644 --- a/tests/lexer/text.arf +++ b/tests/lexer/text.arf @@ -1,4 +1,4 @@ :arf -"hello world!\a\b\f\n\r\t\v\'\"\\" -'\a' '\b' '\f' '\n' '\r' '\t' '\v' '\'' '\"' '\\' -"hello world \x40\u0040\U00000040!" +'hello world!\a\b\f\n\r\t\v\'\\' +'\a' '\b' '\f' '\n' '\r' '\t' '\v' '\'' '\\' +'hello world \x40\u0040\U00000040!'