Lexer now accepts files with no blank line at the end
This commit is contained in:
parent
13a36e43ba
commit
598e3412be
|
@ -56,8 +56,7 @@ Using a C compiler will link the C standard library to your program, which may
|
|||
be useful for building normal user applications.
|
||||
|
||||
## Bugs
|
||||
There are a lot of them. But something to be wary of, especially on Windows:
|
||||
currently, the compiler only excepts files with a blank line at the end.
|
||||
This compiler has *many* bugs. If you come accross any please create an issue.
|
||||
|
||||
## Roadmap
|
||||
|
||||
|
|
|
@ -89,7 +89,7 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
|
||||
token.Pos = this.pos()
|
||||
|
||||
if this.eof == true {
|
||||
if this.eof {
|
||||
token.Type = EOF
|
||||
err = nil
|
||||
return
|
||||
|
@ -105,6 +105,7 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
token.Type = Int
|
||||
for isDigit(this.rune) {
|
||||
appendRune()
|
||||
if this.eof { err = nil; return }
|
||||
if err != nil { return }
|
||||
}
|
||||
if this.rune == '.' {
|
||||
|
@ -131,6 +132,7 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
token.Type = Ident
|
||||
for unicode.IsLetter(this.rune) || isDigit(this.rune) {
|
||||
appendRune()
|
||||
if this.eof { err = nil; return }
|
||||
if err != nil { return }
|
||||
}
|
||||
// TypeIdent
|
||||
|
@ -138,6 +140,7 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
token.Type = TypeIdent
|
||||
for unicode.IsLetter(this.rune) || isDigit(this.rune) {
|
||||
appendRune()
|
||||
if this.eof { err = nil; return }
|
||||
if err != nil { return }
|
||||
}
|
||||
// Int, Float
|
||||
|
@ -157,10 +160,12 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
token.Value += string(result)
|
||||
} else {
|
||||
appendRune()
|
||||
if this.eof { err = nil; return }
|
||||
if err != nil { return }
|
||||
}
|
||||
}
|
||||
err = this.nextRune()
|
||||
if this.eof { err = nil; return }
|
||||
if err != nil { return }
|
||||
// Symbol, Int, Float
|
||||
case this.rune == '-':
|
||||
|
@ -169,12 +174,15 @@ func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
|
|||
if err != nil { return }
|
||||
if isDigit(this.rune) {
|
||||
doNumber()
|
||||
if this.eof { err = nil; return }
|
||||
} else if isSymbol(this.rune) {
|
||||
doSymbol()
|
||||
if this.eof { err = nil; return }
|
||||
}
|
||||
// Symbol
|
||||
case isSymbol(this.rune):
|
||||
doSymbol()
|
||||
if this.eof { err = nil; return }
|
||||
case this.rune == '(':
|
||||
token.Type = LParen
|
||||
appendRune()
|
||||
|
|
|
@ -97,3 +97,31 @@ tok(RParen, ")"),
|
|||
|
||||
tok(EOF, ""),
|
||||
)}
|
||||
|
||||
func TestLexerNoTrailingEOLInt (test *testing.T) {
|
||||
testString(test,
|
||||
`12345`,
|
||||
tok(Int, "12345"),
|
||||
tok(EOF, ""),
|
||||
)}
|
||||
|
||||
func TestLexerNoTrailingEOLString (test *testing.T) {
|
||||
testString(test,
|
||||
`'12345'`,
|
||||
tok(String, "12345"),
|
||||
tok(EOF, ""),
|
||||
)}
|
||||
|
||||
func TestLexerNoTrailingEOLIdent (test *testing.T) {
|
||||
testString(test,
|
||||
`hello`,
|
||||
tok(Ident, "hello"),
|
||||
tok(EOF, ""),
|
||||
)}
|
||||
|
||||
func TestLexerNoTrailingEOLSymbol (test *testing.T) {
|
||||
testString(test,
|
||||
`@#$`,
|
||||
tok(Symbol, "@#$"),
|
||||
tok(EOF, ""),
|
||||
)}
|
||||
|
|
Loading…
Reference in New Issue