Lexer skips over zero runes now
This commit is contained in:
parent
1fd34731ff
commit
3297f6671e
@ -5,6 +5,7 @@ import "io"
|
||||
import "fmt"
|
||||
import "bufio"
|
||||
import "unicode"
|
||||
import "unicode/utf8"
|
||||
import "git.tebibyte.media/fspl/fspl/errors"
|
||||
|
||||
// TokenKind is an enumeration of all tokens the FSPL compiler recognizes.
|
||||
@ -134,7 +135,8 @@ type fsplLexer struct {
|
||||
filename string
|
||||
lineScanner *bufio.Scanner
|
||||
rune rune
|
||||
runeLine []rune
|
||||
line string
|
||||
lineFood string
|
||||
|
||||
offset int
|
||||
row int
|
||||
@ -314,10 +316,11 @@ func (this *fsplLexer) nextInternal () (token Token, err error) {
|
||||
}
|
||||
|
||||
func (this *fsplLexer) nextRune () error {
|
||||
if this.column >= len(this.runeLine) {
|
||||
if this.lineFood == "" {
|
||||
ok := this.lineScanner.Scan()
|
||||
if ok {
|
||||
this.runeLine = []rune(this.lineScanner.Text())
|
||||
this.line = this.lineScanner.Text()
|
||||
this.lineFood = this.line
|
||||
this.rune = '\n'
|
||||
this.column = 0
|
||||
this.row ++
|
||||
@ -331,7 +334,13 @@ func (this *fsplLexer) nextRune () error {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.rune = this.runeLine[this.column]
|
||||
var ch rune
|
||||
var size int
|
||||
for ch == 0 && this.lineFood != "" {
|
||||
ch, size = utf8.DecodeRuneInString(this.lineFood)
|
||||
this.lineFood = this.lineFood[size:]
|
||||
}
|
||||
this.rune = ch
|
||||
this.column ++
|
||||
}
|
||||
|
||||
|
@ -9,7 +9,8 @@ testString (test,
|
||||
+ 'io'
|
||||
+ '../io' customIo`,
|
||||
// input
|
||||
`'5a8353f8cad84604be6029a2575996bc'
|
||||
`
|
||||
'5a8353f8cad84604be6029a2575996bc'
|
||||
+ 'io'
|
||||
+ '../io' customIo
|
||||
`)
|
||||
@ -27,3 +28,13 @@ testString (test,
|
||||
+ 'io' + '../io' customIo
|
||||
`)
|
||||
}
|
||||
|
||||
func TestOnlyUUID (test *testing.T) {
|
||||
testString (test,
|
||||
// correct
|
||||
`'5a8353f8-cad8-4604-be60-29a2575996bc'`,
|
||||
// input
|
||||
`
|
||||
'5a8353f8cad84604be6029a2575996bc'
|
||||
`)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user