Lexer skips over zero runes now
This commit is contained in:
parent
1fd34731ff
commit
3297f6671e
|
@ -5,6 +5,7 @@ import "io"
|
||||||
import "fmt"
|
import "fmt"
|
||||||
import "bufio"
|
import "bufio"
|
||||||
import "unicode"
|
import "unicode"
|
||||||
|
import "unicode/utf8"
|
||||||
import "git.tebibyte.media/fspl/fspl/errors"
|
import "git.tebibyte.media/fspl/fspl/errors"
|
||||||
|
|
||||||
// TokenKind is an enumeration of all tokens the FSPL compiler recognizes.
|
// TokenKind is an enumeration of all tokens the FSPL compiler recognizes.
|
||||||
|
@ -134,7 +135,8 @@ type fsplLexer struct {
|
||||||
filename string
|
filename string
|
||||||
lineScanner *bufio.Scanner
|
lineScanner *bufio.Scanner
|
||||||
rune rune
|
rune rune
|
||||||
runeLine []rune
|
line string
|
||||||
|
lineFood string
|
||||||
|
|
||||||
offset int
|
offset int
|
||||||
row int
|
row int
|
||||||
|
@ -314,10 +316,11 @@ func (this *fsplLexer) nextInternal () (token Token, err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *fsplLexer) nextRune () error {
|
func (this *fsplLexer) nextRune () error {
|
||||||
if this.column >= len(this.runeLine) {
|
if this.lineFood == "" {
|
||||||
ok := this.lineScanner.Scan()
|
ok := this.lineScanner.Scan()
|
||||||
if ok {
|
if ok {
|
||||||
this.runeLine = []rune(this.lineScanner.Text())
|
this.line = this.lineScanner.Text()
|
||||||
|
this.lineFood = this.line
|
||||||
this.rune = '\n'
|
this.rune = '\n'
|
||||||
this.column = 0
|
this.column = 0
|
||||||
this.row ++
|
this.row ++
|
||||||
|
@ -331,7 +334,13 @@ func (this *fsplLexer) nextRune () error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
this.rune = this.runeLine[this.column]
|
var ch rune
|
||||||
|
var size int
|
||||||
|
for ch == 0 && this.lineFood != "" {
|
||||||
|
ch, size = utf8.DecodeRuneInString(this.lineFood)
|
||||||
|
this.lineFood = this.lineFood[size:]
|
||||||
|
}
|
||||||
|
this.rune = ch
|
||||||
this.column ++
|
this.column ++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,8 @@ testString (test,
|
||||||
+ 'io'
|
+ 'io'
|
||||||
+ '../io' customIo`,
|
+ '../io' customIo`,
|
||||||
// input
|
// input
|
||||||
`'5a8353f8cad84604be6029a2575996bc'
|
`
|
||||||
|
'5a8353f8cad84604be6029a2575996bc'
|
||||||
+ 'io'
|
+ 'io'
|
||||||
+ '../io' customIo
|
+ '../io' customIo
|
||||||
`)
|
`)
|
||||||
|
@ -27,3 +28,13 @@ testString (test,
|
||||||
+ 'io' + '../io' customIo
|
+ 'io' + '../io' customIo
|
||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOnlyUUID (test *testing.T) {
|
||||||
|
testString (test,
|
||||||
|
// correct
|
||||||
|
`'5a8353f8-cad8-4604-be60-29a2575996bc'`,
|
||||||
|
// input
|
||||||
|
`
|
||||||
|
'5a8353f8cad84604be6029a2575996bc'
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue