This repository has been archived on 2024-02-27. You can view files and clone it, but cannot push or open issues or pull requests.
arf/lexer/lexer.go

327 lines
7.1 KiB
Go
Raw Normal View History

2022-08-07 13:18:59 -06:00
package lexer
2022-08-09 18:45:06 -06:00
import "io"
import "github.com/sashakoshka/arf/file"
import "github.com/sashakoshka/arf/types"
2022-08-07 13:18:59 -06:00
// LexingOperation holds information about an ongoing lexing operataion.
type LexingOperation struct {
file *file.File
char rune
tokens []Token
2022-08-07 13:18:59 -06:00
}
2022-08-09 18:45:06 -06:00
// Tokenize converts a file into a slice of tokens (lexemes).
func Tokenize (file *file.File) (tokens []Token, err error) {
lexer := LexingOperation { file: file }
err = lexer.tokenize()
tokens = lexer.tokens
2022-08-09 18:45:06 -06:00
// if the lexing operation returned io.EOF, nothing went wrong so we
// return nil for err.
if err == io.EOF {
err = nil
}
return
}
// tokenize converts a file into a slice of tokens (lexemes). It will always
// return a non-nil error, but if nothing went wrong it will return io.EOF.
func (lexer *LexingOperation) tokenize () (err error) {
2022-08-09 18:45:06 -06:00
err = lexer.nextRune()
if err != nil { return }
for {
lowercase := lexer.char >= 'a' && lexer.char <= 'z'
uppercase := lexer.char >= 'A' && lexer.char <= 'Z'
number := lexer.char >= '0' && lexer.char <= '9'
if number {
2022-08-10 23:14:41 -06:00
err = lexer.tokenizeNumberBeginning(false)
if err != nil { return }
2022-08-09 18:45:06 -06:00
} else if lowercase || uppercase {
err = lexer.tokenizeAlphaBeginning()
if err != nil { return }
2022-08-11 00:00:57 -06:00
} else {
2022-08-09 20:18:12 -06:00
err = lexer.tokenizeSymbolBeginning()
if err != nil { return }
2022-08-09 18:45:06 -06:00
}
err = lexer.skipSpaces()
if err != nil { return }
2022-08-09 18:45:06 -06:00
}
2022-08-11 02:47:42 -06:00
if lexer.tokens[len(lexer.tokens) - 1].kind != TokenKindNewline {
lexer.addToken(Token { kind: TokenKindNewline })
}
2022-08-09 18:45:06 -06:00
return
2022-08-07 13:18:59 -06:00
}
func (lexer *LexingOperation) tokenizeAlphaBeginning () (err error) {
got := ""
for {
lowercase := lexer.char >= 'a' && lexer.char <= 'z'
uppercase := lexer.char >= 'A' && lexer.char <= 'Z'
number := lexer.char >= '0' && lexer.char <= '9'
if !lowercase && !uppercase && !number { break }
got += string(lexer.char)
lexer.nextRune()
}
token := Token { kind: TokenKindName, value: got }
if len(got) == 2 {
firstValid := got[0] == 'n' || got[0] == 'r' || got[0] == 'w'
secondValid := got[1] == 'n' || got[1] == 'r' || got[1] == 'w'
if firstValid && secondValid {
token.kind = TokenKindPermission
token.value = types.PermissionFrom(got)
}
}
lexer.addToken(token)
return
}
2022-08-09 20:18:12 -06:00
func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
switch lexer.char {
2022-08-09 23:03:59 -06:00
case '#':
// comment
for lexer.char != '\n' {
err = lexer.nextRune()
if err != nil { return }
2022-08-09 23:03:59 -06:00
}
2022-08-09 20:18:12 -06:00
case '\t':
2022-08-09 23:03:59 -06:00
// indent level
previousToken := lexer.tokens[len(lexer.tokens) - 1]
if !previousToken.Is(TokenKindNewline) ||
!previousToken.Is(TokenKindNewline) {
file.NewError (
lexer.file.Location(), 1,
"tab not used as indent",
file.ErrorKindWarn)
break
}
indentLevel := 0
2022-08-09 20:18:12 -06:00
for lexer.char == '\t' {
indentLevel ++
err = lexer.nextRune()
if err != nil { return }
2022-08-09 20:18:12 -06:00
}
lexer.addToken (Token {
kind: TokenKindIndent,
value: indentLevel,
})
2022-08-09 23:03:59 -06:00
case '\n':
// line break
2022-08-11 02:47:42 -06:00
lastLineEmpty := true
tokenIndex := len(lexer.tokens) - 1
for lexer.tokens[tokenIndex].kind != TokenKindNewline {
if lexer.tokens[tokenIndex].kind != TokenKindIndent {
lastLineEmpty = false
break
}
tokenIndex --
}
if lastLineEmpty {
lexer.tokens = lexer.tokens[:tokenIndex]
}
2022-08-10 09:28:29 -06:00
// TODO: if last line was blank, (ony whitespace) discard.
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindNewline,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case '"':
err = lexer.tokenizeString(false)
2022-08-09 20:18:12 -06:00
case '\'':
err = lexer.tokenizeString(true)
2022-08-09 20:18:12 -06:00
case ':':
lexer.addToken (Token {
kind: TokenKindColon,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case '.':
lexer.addToken (Token {
kind: TokenKindDot,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case '[':
lexer.addToken (Token {
kind: TokenKindLBracket,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case ']':
lexer.addToken (Token {
kind: TokenKindRBracket,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case '{':
lexer.addToken (Token {
kind: TokenKindLBrace,
})
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
case '}':
lexer.addToken (Token {
kind: TokenKindRBrace,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '+':
err = lexer.nextRune()
if err != nil { return }
token := Token { kind: TokenKindPlus }
if lexer.char == '+' {
token.kind = TokenKindIncrement
}
lexer.addToken(token)
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '-':
err = lexer.tokenizeDashBeginning()
2022-08-09 22:48:18 -06:00
case '*':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindAsterisk,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '/':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindSlash,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '@':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindAt,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '!':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindExclamation,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '%':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindPercent,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '~':
2022-08-09 23:03:59 -06:00
lexer.addToken (Token {
kind: TokenKindTilde,
})
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '<':
err = lexer.nextRune()
if err != nil { return }
token := Token { kind: TokenKindLessThan }
if lexer.char == '<' {
token.kind = TokenKindLShift
}
lexer.addToken(token)
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '>':
err = lexer.nextRune()
if err != nil { return }
token := Token { kind: TokenKindGreaterThan }
if lexer.char == '>' {
token.kind = TokenKindRShift
}
lexer.addToken(token)
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '|':
err = lexer.nextRune()
if err != nil { return }
token := Token { kind: TokenKindBinaryOr }
if lexer.char == '|' {
token.kind = TokenKindLogicalOr
}
lexer.addToken(token)
err = lexer.nextRune()
2022-08-09 22:48:18 -06:00
case '&':
err = lexer.nextRune()
if err != nil { return }
token := Token { kind: TokenKindBinaryAnd }
if lexer.char == '&' {
token.kind = TokenKindLogicalAnd
}
lexer.addToken(token)
err = lexer.nextRune()
2022-08-09 20:18:12 -06:00
default:
err = file.NewError (
lexer.file.Location(), 1,
"unexpected symbol character " +
2022-08-09 20:18:12 -06:00
string(lexer.char),
file.ErrorKindError)
return
}
return
}
func (lexer *LexingOperation) tokenizeDashBeginning () (err error) {
2022-08-11 00:10:34 -06:00
err = lexer.nextRune()
if err != nil { return }
if lexer.char == '-' {
2022-08-11 00:10:34 -06:00
token := Token { kind: TokenKindDecrement }
err = lexer.nextRune()
if err != nil { return }
if lexer.char == '-' {
token.kind = TokenKindSeparator
lexer.nextRune()
}
lexer.addToken(token)
} else if lexer.char == '>' {
2022-08-11 00:10:34 -06:00
token := Token { kind: TokenKindReturnDirection }
2022-08-11 00:10:34 -06:00
err = lexer.nextRune()
if err != nil { return }
2022-08-11 00:10:34 -06:00
lexer.addToken(token)
} else if lexer.char >= '0' && lexer.char <= '9' {
lexer.tokenizeNumberBeginning(true)
} else {
token := Token { kind: TokenKindMinus }
lexer.addToken(token)
}
return
}
// addToken adds a new token to the lexer's token slice.
func (lexer *LexingOperation) addToken (token Token) {
lexer.tokens = append(lexer.tokens, token)
}
// skipSpaces skips all space characters (not tabs or newlines)
func (lexer *LexingOperation) skipSpaces () (err error) {
for lexer.char == ' ' {
err = lexer.nextRune()
if err != nil { return }
}
return
}
2022-08-09 18:45:06 -06:00
// nextRune advances the lexer to the next rune in the file.
func (lexer *LexingOperation) nextRune () (err error) {
lexer.char, _, err = lexer.file.ReadRune()
if err != nil && err != io.EOF {
return file.NewError (
lexer.file.Location(), 1,
err.Error(), file.ErrorKindError)
}
2022-08-07 13:18:59 -06:00
return
}