sashakoshka/arf
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

String and rune literals now have correct width

Browse Source
This commit is contained in:
Sasha Koshka 2022-08-18 11:25:40 -04:00
parent a013d4caad
commit bde4bf8493
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lexer/text.go
View File

@ -10,16 +10,19 @@ func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) {
token := lexer.newToken() token := lexer.newToken()
got := "" got := ""
tokenWidth := 2
beginning := lexer.file.Location(1) beginning := lexer.file.Location(1)
for { for {
// TODO: add hexadecimal escape codes
if lexer.char == '\\' { if lexer.char == '\\' {
err = lexer.nextRune() err = lexer.nextRune()
tokenWidth ++
if err != nil { return } if err != nil { return }
var actual rune var actual rune
actual, err = lexer.getEscapeSequence() var amountRead int
actual, amountRead, err = lexer.getEscapeSequence()
tokenWidth += amountRead
if err != nil { return } if err != nil { return }
got += string(actual) got += string(actual)
@ -27,6 +30,7 @@ func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) {
got += string(lexer.char) got += string(lexer.char)
err = lexer.nextRune() err = lexer.nextRune()
tokenWidth ++
if err != nil { return } if err != nil { return }
} }
@ -57,6 +61,7 @@ func (lexer *LexingOperation) tokenizeString (isRuneLiteral bool) (err error) {
token.value = got token.value = got
} }
token.location.SetWidth(tokenWidth)
lexer.addToken(token) lexer.addToken(token)
return return
} }
@ -77,16 +82,22 @@ var escapeSequenceMap = map[rune] rune {
} }
// getEscapeSequence reads an escape sequence in a string or rune literal. // getEscapeSequence reads an escape sequence in a string or rune literal.
func (lexer *LexingOperation) getEscapeSequence () (result rune, err error) { func (lexer *LexingOperation) getEscapeSequence () (
result rune,
amountRead int,
err error,
) {
result, exists := escapeSequenceMap[lexer.char] result, exists := escapeSequenceMap[lexer.char]
if exists { if exists {
err = lexer.nextRune() err = lexer.nextRune()
amountRead ++
return return
} else if lexer.char >= '0' && lexer.char <= '7' { } else if lexer.char >= '0' && lexer.char <= '7' {
// octal escape sequence // octal escape sequence
number := string(lexer.char) number := string(lexer.char)
err = lexer.nextRune() err = lexer.nextRune()
amountRead ++
if err != nil { return } if err != nil { return }
for len(number) < 3 { for len(number) < 3 {
@ -95,6 +106,7 @@ func (lexer *LexingOperation) getEscapeSequence () (result rune, err error) {
number += string(lexer.char) number += string(lexer.char)
err = lexer.nextRune() err = lexer.nextRune()
amountRead ++
if err != nil { return } if err != nil { return }
} }
@ -118,6 +130,7 @@ func (lexer *LexingOperation) getEscapeSequence () (result rune, err error) {
number := "" number := ""
err = lexer.nextRune() err = lexer.nextRune()
amountRead ++
if err != nil { return } if err != nil { return }
for len(number) < want { for len(number) < want {
@ -129,6 +142,7 @@ func (lexer *LexingOperation) getEscapeSequence () (result rune, err error) {
number += string(lexer.char) number += string(lexer.char)
err = lexer.nextRune() err = lexer.nextRune()
amountRead ++
if err != nil { return } if err != nil { return }
} }