440 lines
10 KiB
Go
440 lines
10 KiB
Go
package lexer
|
|
|
|
import "io"
|
|
import "git.tebibyte.media/arf/arf/file"
|
|
import "git.tebibyte.media/arf/arf/types"
|
|
import "git.tebibyte.media/arf/arf/infoerr"
|
|
|
|
// LexingOperation holds information about an ongoing lexing operataion.
|
|
type LexingOperation struct {
|
|
file *file.File
|
|
char rune
|
|
tokens []Token
|
|
}
|
|
|
|
// Tokenize converts a file into a slice of tokens (lexemes).
|
|
func Tokenize (file *file.File) (tokens []Token, err error) {
|
|
lexer := LexingOperation { file: file }
|
|
err = lexer.tokenize()
|
|
tokens = lexer.tokens
|
|
|
|
// if the lexing operation returned io.EOF, nothing went wrong so we
|
|
// return nil for err.
|
|
if err == io.EOF {
|
|
err = nil
|
|
}
|
|
return
|
|
}
|
|
|
|
// tokenize converts a file into a slice of tokens (lexemes). It will always
|
|
// return a non-nil error, but if nothing went wrong it will return io.EOF.
|
|
func (lexer *LexingOperation) tokenize () (err error) {
|
|
// check to see if the beginning of the file says :arf
|
|
var shebangCheck = []rune(":arf\n")
|
|
for index := 0; index < 5; index ++ {
|
|
err = lexer.nextRune()
|
|
|
|
if err != nil || shebangCheck[index] != lexer.char {
|
|
err = infoerr.NewError (
|
|
lexer.file.Location(1),
|
|
"not an arf file",
|
|
infoerr.ErrorKindError)
|
|
return
|
|
}
|
|
}
|
|
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
|
|
for {
|
|
lowercase := lexer.char >= 'a' && lexer.char <= 'z'
|
|
uppercase := lexer.char >= 'A' && lexer.char <= 'Z'
|
|
number := lexer.char >= '0' && lexer.char <= '9'
|
|
|
|
if number {
|
|
err = lexer.tokenizeNumberBeginning(false)
|
|
if err != nil { return }
|
|
} else if lowercase || uppercase {
|
|
err = lexer.tokenizeAlphaBeginning()
|
|
if err != nil { return }
|
|
} else {
|
|
err = lexer.tokenizeSymbolBeginning()
|
|
if err != nil { return }
|
|
}
|
|
|
|
err = lexer.skipSpaces()
|
|
if err != nil { return }
|
|
}
|
|
|
|
if lexer.tokens[len(lexer.tokens) - 1].kind != TokenKindNewline {
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindNewline
|
|
lexer.addToken(token)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (lexer *LexingOperation) tokenizeAlphaBeginning () (err error) {
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindName
|
|
|
|
got := ""
|
|
|
|
for {
|
|
lowercase := lexer.char >= 'a' && lexer.char <= 'z'
|
|
uppercase := lexer.char >= 'A' && lexer.char <= 'Z'
|
|
number := lexer.char >= '0' && lexer.char <= '9'
|
|
if !lowercase && !uppercase && !number { break }
|
|
|
|
got += string(lexer.char)
|
|
|
|
lexer.nextRune()
|
|
}
|
|
|
|
token.value = got
|
|
token.location.SetWidth(len(got))
|
|
|
|
if len(got) == 2 {
|
|
permission, isPermission := types.PermissionFrom(got)
|
|
|
|
if isPermission {
|
|
token.kind = TokenKindPermission
|
|
token.value = permission
|
|
}
|
|
}
|
|
|
|
lexer.addToken(token)
|
|
|
|
return
|
|
}
|
|
|
|
func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
|
|
switch lexer.char {
|
|
case '#':
|
|
// comment
|
|
for lexer.char != '\n' {
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
}
|
|
case '\t':
|
|
// indent level
|
|
previousToken := lexer.tokens[len(lexer.tokens) - 1]
|
|
|
|
if !previousToken.Is(TokenKindNewline) {
|
|
err = lexer.nextRune()
|
|
|
|
infoerr.NewError (
|
|
lexer.file.Location(1),
|
|
"tab not used as indent",
|
|
infoerr.ErrorKindWarn).Print()
|
|
return
|
|
}
|
|
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindIndent
|
|
|
|
// eat up tabs while increasing the indent level
|
|
indentLevel := 0
|
|
for lexer.char == '\t' {
|
|
indentLevel ++
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
}
|
|
|
|
token.value = indentLevel
|
|
token.location.SetWidth(indentLevel)
|
|
lexer.addToken(token)
|
|
case '\n':
|
|
// line break
|
|
|
|
// if the last line is empty, discard it
|
|
lastLineEmpty := true
|
|
tokenIndex := len(lexer.tokens) - 1
|
|
for lexer.tokens[tokenIndex].kind != TokenKindNewline {
|
|
if lexer.tokens[tokenIndex].kind != TokenKindIndent {
|
|
lastLineEmpty = false
|
|
break
|
|
}
|
|
tokenIndex --
|
|
}
|
|
|
|
if lastLineEmpty {
|
|
lexer.tokens = lexer.tokens[:tokenIndex]
|
|
}
|
|
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindNewline
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '"':
|
|
err = lexer.tokenizeString(false)
|
|
case '\'':
|
|
err = lexer.tokenizeString(true)
|
|
case ':':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindColon
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '.':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindDot
|
|
if lexer.char == '.' {
|
|
token.kind = TokenKindElipsis
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case ',':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindComma
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '[':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindLBracket
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case ']':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindRBracket
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '{':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindLBrace
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '}':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindRBrace
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '+':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindPlus
|
|
if lexer.char == '+' {
|
|
token.kind = TokenKindIncrement
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '-':
|
|
err = lexer.tokenizeDashBeginning()
|
|
case '*':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindAsterisk
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '/':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindSlash
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '@':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindAt
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '!':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindExclamation
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindNotEqualTo
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '%':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindPercent
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindPercentAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '~':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindTilde
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindTildeAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '=':
|
|
token := lexer.newToken()
|
|
token.kind = TokenKindEqualTo
|
|
lexer.addToken(token)
|
|
err = lexer.nextRune()
|
|
case '<':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindLessThan
|
|
if lexer.char == '<' {
|
|
token.kind = TokenKindLShift
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindLShiftAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(3)
|
|
}
|
|
} else if lexer.char == '=' {
|
|
token.kind = TokenKindLessThanEqualTo
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '>':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindGreaterThan
|
|
if lexer.char == '>' {
|
|
token.kind = TokenKindRShift
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindRShiftAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(3)
|
|
}
|
|
} else if lexer.char == '=' {
|
|
token.kind = TokenKindGreaterThanEqualTo
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '|':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindBinaryOr
|
|
if lexer.char == '|' {
|
|
token.kind = TokenKindLogicalOr
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
} else if lexer.char == '=' {
|
|
token.kind = TokenKindBinaryOrAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '&':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindBinaryAnd
|
|
if lexer.char == '&' {
|
|
token.kind = TokenKindLogicalAnd
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
} else if lexer.char == '=' {
|
|
token.kind = TokenKindBinaryAndAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
case '^':
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
token.kind = TokenKindBinaryXor
|
|
if lexer.char == '=' {
|
|
token.kind = TokenKindBinaryXorAssignment
|
|
err = lexer.nextRune()
|
|
token.location.SetWidth(2)
|
|
}
|
|
lexer.addToken(token)
|
|
default:
|
|
err = infoerr.NewError (
|
|
lexer.file.Location(1),
|
|
"unexpected symbol character " +
|
|
string(lexer.char),
|
|
infoerr.ErrorKindError)
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (lexer *LexingOperation) tokenizeDashBeginning () (err error) {
|
|
token := lexer.newToken()
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
|
|
if lexer.char == '-' {
|
|
token.kind = TokenKindDecrement
|
|
token.location.SetWidth(2)
|
|
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
|
|
if lexer.char == '-' {
|
|
token.kind = TokenKindSeparator
|
|
lexer.nextRune()
|
|
token.location.SetWidth(3)
|
|
}
|
|
lexer.addToken(token)
|
|
} else if lexer.char == '>' {
|
|
token.kind = TokenKindReturnDirection
|
|
token.location.SetWidth(2)
|
|
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
|
|
lexer.addToken(token)
|
|
} else if lexer.char >= '0' && lexer.char <= '9' {
|
|
lexer.tokenizeNumberBeginning(true)
|
|
} else {
|
|
token.kind = TokenKindMinus
|
|
lexer.addToken(token)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// newToken creates a new token from the lexer's current position in the file.
|
|
func (lexer *LexingOperation) newToken () (token Token) {
|
|
return Token { location: lexer.file.Location(1) }
|
|
}
|
|
|
|
// addToken adds a new token to the lexer's token slice.
|
|
func (lexer *LexingOperation) addToken (token Token) {
|
|
lexer.tokens = append(lexer.tokens, token)
|
|
}
|
|
|
|
// skipSpaces skips all space characters (not tabs or newlines)
|
|
func (lexer *LexingOperation) skipSpaces () (err error) {
|
|
for lexer.char == ' ' {
|
|
err = lexer.nextRune()
|
|
if err != nil { return }
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// nextRune advances the lexer to the next rune in the file.
|
|
func (lexer *LexingOperation) nextRune () (err error) {
|
|
lexer.char, _, err = lexer.file.ReadRune()
|
|
if err != nil && err != io.EOF {
|
|
return infoerr.NewError (
|
|
lexer.file.Location(1),
|
|
err.Error(), infoerr.ErrorKindError)
|
|
}
|
|
return
|
|
}
|