This repository has been archived on 2024-02-27. You can view files and clone it, but cannot push or open issues or pull requests.
arf/lexer/lexer.go

20 lines
501 B
Go

package lexer
import "github.com/sashakoshka/arf/file"
// LexingOperation holds information about an ongoing lexing operataion.
type LexingOperation struct {
file *file.File
}
// Tokenize converts a file into a slice of tokens (lexemes)
func Tokenize (file *file.File) (tokens []Token) {
lexer := LexingOperation { }
return lexer.tokenize(file)
}
// tokenize converts a file into a slice of tokens (lexemes)
func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) {
return
}